2024-11-24 20:46:52,344 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-24 20:46:52,354 main DEBUG Took 0.008493 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 20:46:52,354 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 20:46:52,355 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 20:46:52,356 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 20:46:52,357 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,363 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 20:46:52,374 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,377 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,378 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,378 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,379 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,380 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,382 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,383 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,384 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,385 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,385 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,386 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,386 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,387 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,387 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,388 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,388 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:52,389 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,389 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 20:46:52,391 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:52,392 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 20:46:52,395 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 20:46:52,395 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 20:46:52,397 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 20:46:52,398 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 20:46:52,407 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 20:46:52,409 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 20:46:52,410 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 20:46:52,411 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 20:46:52,411 main DEBUG createAppenders(={Console}) 2024-11-24 20:46:52,412 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-24 20:46:52,412 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-24 20:46:52,412 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-24 20:46:52,413 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 20:46:52,413 main DEBUG OutputStream closed 2024-11-24 20:46:52,413 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 20:46:52,413 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 20:46:52,414 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-24 20:46:52,481 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 20:46:52,483 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 20:46:52,484 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 20:46:52,485 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 20:46:52,486 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 20:46:52,486 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 20:46:52,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 20:46:52,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 20:46:52,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 20:46:52,488 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 20:46:52,488 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 20:46:52,488 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 20:46:52,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 20:46:52,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 20:46:52,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 20:46:52,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 20:46:52,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 20:46:52,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 20:46:52,493 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 20:46:52,493 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-24 20:46:52,493 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 20:46:52,494 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-24T20:46:52,506 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-24 20:46:52,508 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 20:46:52,509 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T20:46:52,563 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=545, ProcessCount=11, AvailableMemoryMB=10504 2024-11-24T20:46:52,802 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae 2024-11-24T20:46:53,346 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T20:46:53,444 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T20:46:53,498 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-24T20:46:53,546 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481213537 2024-11-24T20:46:53,605 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481213537 2024-11-24T20:46:53,671 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:53,677 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-24T20:46:53,678 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-24T20:46:53,713 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8fabde2134d9c90b17ff9d84ca456f90, NAME => 'testLockupAroundBadAssignSync,,1732481213690.8fabde2134d9c90b17ff9d84ca456f90.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae 2024-11-24T20:46:53,757 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T20:46:53,762 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1732481213690.8fabde2134d9c90b17ff9d84ca456f90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:46:53,849 INFO [StoreOpener-8fabde2134d9c90b17ff9d84ca456f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region 8fabde2134d9c90b17ff9d84ca456f90 2024-11-24T20:46:53,885 INFO [StoreOpener-8fabde2134d9c90b17ff9d84ca456f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fabde2134d9c90b17ff9d84ca456f90 columnFamilyName MyCF 2024-11-24T20:46:53,893 DEBUG [StoreOpener-8fabde2134d9c90b17ff9d84ca456f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:53,901 INFO [StoreOpener-8fabde2134d9c90b17ff9d84ca456f90-1 {}] regionserver.HStore(327): Store=8fabde2134d9c90b17ff9d84ca456f90/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:53,905 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8fabde2134d9c90b17ff9d84ca456f90 2024-11-24T20:46:53,913 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/data/default/testLockupAroundBadAssignSync/8fabde2134d9c90b17ff9d84ca456f90 2024-11-24T20:46:53,914 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/data/default/testLockupAroundBadAssignSync/8fabde2134d9c90b17ff9d84ca456f90 2024-11-24T20:46:53,918 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8fabde2134d9c90b17ff9d84ca456f90 2024-11-24T20:46:53,919 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8fabde2134d9c90b17ff9d84ca456f90 2024-11-24T20:46:53,928 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8fabde2134d9c90b17ff9d84ca456f90 2024-11-24T20:46:53,939 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/data/default/testLockupAroundBadAssignSync/8fabde2134d9c90b17ff9d84ca456f90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:46:53,941 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8fabde2134d9c90b17ff9d84ca456f90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68302267, jitterRate=0.017783090472221375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T20:46:53,950 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T20:46:53,967 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8fabde2134d9c90b17ff9d84ca456f90: Writing region info on filesystem at 1732481213817Initializing all the Stores at 1732481213819 (+2 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1732481213820 (+1 ms)Cleaning up temporary data from old regions at 1732481213920 (+100 ms)Region opened successfully at 1732481213964 (+44 ms) 2024-11-24T20:46:54,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481213537) roll requested 2024-11-24T20:46:54,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214021 2024-11-24T20:46:54,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481213537 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214021 2024-11-24T20:46:54,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214021) roll requested 2024-11-24T20:46:54,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214037 2024-11-24T20:46:54,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214037 2024-11-24T20:46:54,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214037) roll requested 2024-11-24T20:46:54,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214054 2024-11-24T20:46:54,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214054 2024-11-24T20:46:54,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214054) roll requested 2024-11-24T20:46:54,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214067 2024-11-24T20:46:54,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214067 2024-11-24T20:46:54,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214067) roll requested 2024-11-24T20:46:54,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214078 2024-11-24T20:46:54,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214078 2024-11-24T20:46:54,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214078) roll requested 2024-11-24T20:46:54,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214088 2024-11-24T20:46:54,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214088 2024-11-24T20:46:54,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214088) roll requested 2024-11-24T20:46:54,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214098 2024-11-24T20:46:54,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214098 2024-11-24T20:46:54,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214098) roll requested 2024-11-24T20:46:54,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214108 2024-11-24T20:46:54,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214108 2024-11-24T20:46:54,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214108) roll requested 2024-11-24T20:46:54,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214119 2024-11-24T20:46:54,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214119 2024-11-24T20:46:54,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214119) roll requested 2024-11-24T20:46:54,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214129 2024-11-24T20:46:54,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214129 2024-11-24T20:46:54,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214129) roll requested 2024-11-24T20:46:54,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214139 2024-11-24T20:46:54,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214139 2024-11-24T20:46:54,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214139) roll requested 2024-11-24T20:46:54,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214148 2024-11-24T20:46:54,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214148 2024-11-24T20:46:54,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214148) roll requested 2024-11-24T20:46:54,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214158 2024-11-24T20:46:54,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214158 2024-11-24T20:46:54,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214158) roll requested 2024-11-24T20:46:54,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214169 2024-11-24T20:46:54,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214169 2024-11-24T20:46:54,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214169) roll requested 2024-11-24T20:46:54,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214180 2024-11-24T20:46:54,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214180 2024-11-24T20:46:54,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214180) roll requested 2024-11-24T20:46:54,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214189 2024-11-24T20:46:54,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214189 2024-11-24T20:46:54,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214189) roll requested 2024-11-24T20:46:54,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214200 2024-11-24T20:46:54,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214200 2024-11-24T20:46:54,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214200) roll requested 2024-11-24T20:46:54,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214211 2024-11-24T20:46:54,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214211 2024-11-24T20:46:54,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214211) roll requested 2024-11-24T20:46:54,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214223 2024-11-24T20:46:54,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214223 2024-11-24T20:46:54,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214223) roll requested 2024-11-24T20:46:54,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214236 2024-11-24T20:46:54,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214236 2024-11-24T20:46:54,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214236) roll requested 2024-11-24T20:46:54,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214249 2024-11-24T20:46:54,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214249 2024-11-24T20:46:54,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214249) roll requested 2024-11-24T20:46:54,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214262 2024-11-24T20:46:54,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214262 2024-11-24T20:46:54,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214262) roll requested 2024-11-24T20:46:54,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214273 2024-11-24T20:46:54,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214273 2024-11-24T20:46:54,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214273) roll requested 2024-11-24T20:46:54,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214289 2024-11-24T20:46:54,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214289 2024-11-24T20:46:54,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214289) roll requested 2024-11-24T20:46:54,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214303 2024-11-24T20:46:54,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214303 2024-11-24T20:46:54,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214303) roll requested 2024-11-24T20:46:54,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214315 2024-11-24T20:46:54,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214315 2024-11-24T20:46:54,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214315) roll requested 2024-11-24T20:46:54,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214325 2024-11-24T20:46:54,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214325 2024-11-24T20:46:54,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214325) roll requested 2024-11-24T20:46:54,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214333 2024-11-24T20:46:54,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214333 2024-11-24T20:46:54,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214333) roll requested 2024-11-24T20:46:54,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214344 2024-11-24T20:46:54,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214344 2024-11-24T20:46:54,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214344) roll requested 2024-11-24T20:46:54,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214354 2024-11-24T20:46:54,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214354 2024-11-24T20:46:54,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214354) roll requested 2024-11-24T20:46:54,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214363 2024-11-24T20:46:54,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214363 2024-11-24T20:46:54,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214363) roll requested 2024-11-24T20:46:54,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214371 2024-11-24T20:46:54,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214371 2024-11-24T20:46:54,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214371) roll requested 2024-11-24T20:46:54,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214379 2024-11-24T20:46:54,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214379 2024-11-24T20:46:54,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214379) roll requested 2024-11-24T20:46:54,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214387 2024-11-24T20:46:54,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214387 2024-11-24T20:46:54,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214387) roll requested 2024-11-24T20:46:54,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214395 2024-11-24T20:46:54,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214395 2024-11-24T20:46:54,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214395) roll requested 2024-11-24T20:46:54,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214404 2024-11-24T20:46:54,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214404 2024-11-24T20:46:54,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214404) roll requested 2024-11-24T20:46:54,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214415 2024-11-24T20:46:54,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214415 2024-11-24T20:46:54,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214415) roll requested 2024-11-24T20:46:54,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214424 2024-11-24T20:46:54,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214424 2024-11-24T20:46:54,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214424) roll requested 2024-11-24T20:46:54,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214432 2024-11-24T20:46:54,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214432 2024-11-24T20:46:54,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214432) roll requested 2024-11-24T20:46:54,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214445 2024-11-24T20:46:54,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214445 2024-11-24T20:46:54,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214445) roll requested 2024-11-24T20:46:54,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214452 2024-11-24T20:46:54,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214452 2024-11-24T20:46:54,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214452) roll requested 2024-11-24T20:46:54,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214461 2024-11-24T20:46:54,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214461 2024-11-24T20:46:54,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214461) roll requested 2024-11-24T20:46:54,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214470 2024-11-24T20:46:54,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214470 2024-11-24T20:46:54,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214470) roll requested 2024-11-24T20:46:54,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214480 2024-11-24T20:46:54,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214480 2024-11-24T20:46:54,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214480) roll requested 2024-11-24T20:46:54,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214489 2024-11-24T20:46:54,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214489 2024-11-24T20:46:54,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214489) roll requested 2024-11-24T20:46:54,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214496 2024-11-24T20:46:54,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214496 2024-11-24T20:46:54,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214496) roll requested 2024-11-24T20:46:54,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214504 2024-11-24T20:46:54,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214504 2024-11-24T20:46:54,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214504) roll requested 2024-11-24T20:46:54,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214514 2024-11-24T20:46:54,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214514 2024-11-24T20:46:54,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214514) roll requested 2024-11-24T20:46:54,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214522 2024-11-24T20:46:54,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214522 2024-11-24T20:46:54,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214522) roll requested 2024-11-24T20:46:54,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214534 2024-11-24T20:46:54,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214534 2024-11-24T20:46:54,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214534) roll requested 2024-11-24T20:46:54,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214542 2024-11-24T20:46:54,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214542 2024-11-24T20:46:54,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214542) roll requested 2024-11-24T20:46:54,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214550 2024-11-24T20:46:54,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214550 2024-11-24T20:46:54,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214550) roll requested 2024-11-24T20:46:54,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214557 2024-11-24T20:46:54,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214557 2024-11-24T20:46:54,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214557) roll requested 2024-11-24T20:46:54,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214566 2024-11-24T20:46:54,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214566 2024-11-24T20:46:54,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214566) roll requested 2024-11-24T20:46:54,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214576 2024-11-24T20:46:54,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214576 2024-11-24T20:46:54,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214576) roll requested 2024-11-24T20:46:54,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214588 2024-11-24T20:46:54,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214588 2024-11-24T20:46:54,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214588) roll requested 2024-11-24T20:46:54,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214599 2024-11-24T20:46:54,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214599 2024-11-24T20:46:54,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214599) roll requested 2024-11-24T20:46:54,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214609 2024-11-24T20:46:54,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214609 2024-11-24T20:46:54,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214609) roll requested 2024-11-24T20:46:54,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214617 2024-11-24T20:46:54,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214617 2024-11-24T20:46:54,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214617) roll requested 2024-11-24T20:46:54,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214624 2024-11-24T20:46:54,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214624 2024-11-24T20:46:54,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214624) roll requested 2024-11-24T20:46:54,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214631 2024-11-24T20:46:54,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214631 2024-11-24T20:46:54,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214631) roll requested 2024-11-24T20:46:54,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214639 2024-11-24T20:46:54,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214639 2024-11-24T20:46:54,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214639) roll requested 2024-11-24T20:46:54,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214648 2024-11-24T20:46:54,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214648 2024-11-24T20:46:54,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214648) roll requested 2024-11-24T20:46:54,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214660 2024-11-24T20:46:54,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214660 2024-11-24T20:46:54,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214660) roll requested 2024-11-24T20:46:54,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214669 2024-11-24T20:46:54,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214669 2024-11-24T20:46:54,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214669) roll requested 2024-11-24T20:46:54,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214677 2024-11-24T20:46:54,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214677 2024-11-24T20:46:54,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214677) roll requested 2024-11-24T20:46:54,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214685 2024-11-24T20:46:54,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214685 2024-11-24T20:46:54,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214685) roll requested 2024-11-24T20:46:54,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214692 2024-11-24T20:46:54,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214692 2024-11-24T20:46:54,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214692) roll requested 2024-11-24T20:46:54,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214699 2024-11-24T20:46:54,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214699 2024-11-24T20:46:54,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214699) roll requested 2024-11-24T20:46:54,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214707 2024-11-24T20:46:54,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214707 2024-11-24T20:46:54,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214707) roll requested 2024-11-24T20:46:54,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214714 2024-11-24T20:46:54,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214714 2024-11-24T20:46:54,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214714) roll requested 2024-11-24T20:46:54,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214721 2024-11-24T20:46:54,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214721 2024-11-24T20:46:54,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214721) roll requested 2024-11-24T20:46:54,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214729 2024-11-24T20:46:54,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214729 2024-11-24T20:46:54,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214729) roll requested 2024-11-24T20:46:54,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214738 2024-11-24T20:46:54,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214738 2024-11-24T20:46:54,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214738) roll requested 2024-11-24T20:46:54,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214747 2024-11-24T20:46:54,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214747 2024-11-24T20:46:54,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214747) roll requested 2024-11-24T20:46:54,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214757 2024-11-24T20:46:54,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214757 2024-11-24T20:46:54,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214757) roll requested 2024-11-24T20:46:54,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214767 2024-11-24T20:46:54,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214767 2024-11-24T20:46:54,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214767) roll requested 2024-11-24T20:46:54,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214775 2024-11-24T20:46:54,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214775 2024-11-24T20:46:54,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214775) roll requested 2024-11-24T20:46:54,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214785 2024-11-24T20:46:54,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214785 2024-11-24T20:46:54,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214785) roll requested 2024-11-24T20:46:54,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214796 2024-11-24T20:46:54,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214796 2024-11-24T20:46:54,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214796) roll requested 2024-11-24T20:46:54,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214806 2024-11-24T20:46:54,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214806 2024-11-24T20:46:54,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214806) roll requested 2024-11-24T20:46:54,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214813 2024-11-24T20:46:54,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214813 2024-11-24T20:46:54,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214813) roll requested 2024-11-24T20:46:54,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214823 2024-11-24T20:46:54,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214823 2024-11-24T20:46:54,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214823) roll requested 2024-11-24T20:46:54,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214832 2024-11-24T20:46:54,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214832 2024-11-24T20:46:54,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214832) roll requested 2024-11-24T20:46:54,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214841 2024-11-24T20:46:54,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214841 2024-11-24T20:46:54,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214841) roll requested 2024-11-24T20:46:54,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214847 2024-11-24T20:46:54,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214847 2024-11-24T20:46:54,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214847) roll requested 2024-11-24T20:46:54,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214853 2024-11-24T20:46:54,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214853 2024-11-24T20:46:54,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214853) roll requested 2024-11-24T20:46:54,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214863 2024-11-24T20:46:54,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214863 2024-11-24T20:46:54,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214863) roll requested 2024-11-24T20:46:54,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214874 2024-11-24T20:46:54,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214874 2024-11-24T20:46:54,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214874) roll requested 2024-11-24T20:46:54,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214882 2024-11-24T20:46:54,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214882 2024-11-24T20:46:54,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214882) roll requested 2024-11-24T20:46:54,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214891 2024-11-24T20:46:54,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214891 2024-11-24T20:46:54,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214891) roll requested 2024-11-24T20:46:54,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214901 2024-11-24T20:46:54,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214901 2024-11-24T20:46:54,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214901) roll requested 2024-11-24T20:46:54,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214913 2024-11-24T20:46:54,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214913 2024-11-24T20:46:54,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214913) roll requested 2024-11-24T20:46:54,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214923 2024-11-24T20:46:54,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214923 2024-11-24T20:46:54,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214923) roll requested 2024-11-24T20:46:54,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214934 2024-11-24T20:46:54,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214934 2024-11-24T20:46:54,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214934) roll requested 2024-11-24T20:46:54,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214944 2024-11-24T20:46:54,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214944 2024-11-24T20:46:54,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214944) roll requested 2024-11-24T20:46:54,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214953 2024-11-24T20:46:54,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214953 2024-11-24T20:46:54,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214953) roll requested 2024-11-24T20:46:54,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214964 2024-11-24T20:46:54,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214964 2024-11-24T20:46:54,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214964) roll requested 2024-11-24T20:46:54,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214973 2024-11-24T20:46:54,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214973 2024-11-24T20:46:54,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214973) roll requested 2024-11-24T20:46:54,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214983 2024-11-24T20:46:54,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:54,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214983 2024-11-24T20:46:54,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:54,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:54,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214983) roll requested 2024-11-24T20:46:54,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481214993 2024-11-24T20:46:55,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214993 2024-11-24T20:46:55,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481214993) roll requested 2024-11-24T20:46:55,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215009 2024-11-24T20:46:55,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481214993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215009 2024-11-24T20:46:55,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215009) roll requested 2024-11-24T20:46:55,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215016 2024-11-24T20:46:55,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215016 2024-11-24T20:46:55,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215016) roll requested 2024-11-24T20:46:55,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215022 2024-11-24T20:46:55,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215022 2024-11-24T20:46:55,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215022) roll requested 2024-11-24T20:46:55,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215029 2024-11-24T20:46:55,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215029 2024-11-24T20:46:55,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215029) roll requested 2024-11-24T20:46:55,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215038 2024-11-24T20:46:55,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215038 2024-11-24T20:46:55,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215038) roll requested 2024-11-24T20:46:55,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215049 2024-11-24T20:46:55,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215049 2024-11-24T20:46:55,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215049) roll requested 2024-11-24T20:46:55,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215059 2024-11-24T20:46:55,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215059 2024-11-24T20:46:55,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215059) roll requested 2024-11-24T20:46:55,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215065 2024-11-24T20:46:55,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215065 2024-11-24T20:46:55,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215065) roll requested 2024-11-24T20:46:55,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215072 2024-11-24T20:46:55,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215072 2024-11-24T20:46:55,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215072) roll requested 2024-11-24T20:46:55,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215080 2024-11-24T20:46:55,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215080 2024-11-24T20:46:55,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215080) roll requested 2024-11-24T20:46:55,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215089 2024-11-24T20:46:55,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215089 2024-11-24T20:46:55,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215089) roll requested 2024-11-24T20:46:55,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215101 2024-11-24T20:46:55,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215101 2024-11-24T20:46:55,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215101) roll requested 2024-11-24T20:46:55,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215113 2024-11-24T20:46:55,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215113 2024-11-24T20:46:55,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215113) roll requested 2024-11-24T20:46:55,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215123 2024-11-24T20:46:55,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215123 2024-11-24T20:46:55,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215123) roll requested 2024-11-24T20:46:55,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215130 2024-11-24T20:46:55,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215130 2024-11-24T20:46:55,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215130) roll requested 2024-11-24T20:46:55,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215140 2024-11-24T20:46:55,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215140 2024-11-24T20:46:55,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215140) roll requested 2024-11-24T20:46:55,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215152 2024-11-24T20:46:55,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215152 2024-11-24T20:46:55,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215152) roll requested 2024-11-24T20:46:55,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215158 2024-11-24T20:46:55,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215158 2024-11-24T20:46:55,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215158) roll requested 2024-11-24T20:46:55,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215168 2024-11-24T20:46:55,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215168 2024-11-24T20:46:55,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215168) roll requested 2024-11-24T20:46:55,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215176 2024-11-24T20:46:55,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215176 2024-11-24T20:46:55,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215176) roll requested 2024-11-24T20:46:55,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215183 2024-11-24T20:46:55,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215183 2024-11-24T20:46:55,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215183) roll requested 2024-11-24T20:46:55,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215192 2024-11-24T20:46:55,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215192 2024-11-24T20:46:55,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215192) roll requested 2024-11-24T20:46:55,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215201 2024-11-24T20:46:55,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215201 2024-11-24T20:46:55,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215201) roll requested 2024-11-24T20:46:55,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215209 2024-11-24T20:46:55,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215209 2024-11-24T20:46:55,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215209) roll requested 2024-11-24T20:46:55,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215217 2024-11-24T20:46:55,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215217 2024-11-24T20:46:55,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215217) roll requested 2024-11-24T20:46:55,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215226 2024-11-24T20:46:55,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215226 2024-11-24T20:46:55,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215226) roll requested 2024-11-24T20:46:55,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215235 2024-11-24T20:46:55,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215235 2024-11-24T20:46:55,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215235) roll requested 2024-11-24T20:46:55,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215241 2024-11-24T20:46:55,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215241 2024-11-24T20:46:55,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215241) roll requested 2024-11-24T20:46:55,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215249 2024-11-24T20:46:55,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215249 2024-11-24T20:46:55,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215249) roll requested 2024-11-24T20:46:55,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215257 2024-11-24T20:46:55,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215257 2024-11-24T20:46:55,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215257) roll requested 2024-11-24T20:46:55,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215272 2024-11-24T20:46:55,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215272 2024-11-24T20:46:55,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215272) roll requested 2024-11-24T20:46:55,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215278 2024-11-24T20:46:55,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215278 2024-11-24T20:46:55,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215278) roll requested 2024-11-24T20:46:55,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215286 2024-11-24T20:46:55,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215286 2024-11-24T20:46:55,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215286) roll requested 2024-11-24T20:46:55,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215292 2024-11-24T20:46:55,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215292 2024-11-24T20:46:55,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215292) roll requested 2024-11-24T20:46:55,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215301 2024-11-24T20:46:55,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215301 2024-11-24T20:46:55,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215301) roll requested 2024-11-24T20:46:55,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215308 2024-11-24T20:46:55,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215308 2024-11-24T20:46:55,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215308) roll requested 2024-11-24T20:46:55,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215317 2024-11-24T20:46:55,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215317 2024-11-24T20:46:55,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215317) roll requested 2024-11-24T20:46:55,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215328 2024-11-24T20:46:55,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215328 2024-11-24T20:46:55,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215328) roll requested 2024-11-24T20:46:55,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215339 2024-11-24T20:46:55,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215339 2024-11-24T20:46:55,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215339) roll requested 2024-11-24T20:46:55,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215350 2024-11-24T20:46:55,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215350 2024-11-24T20:46:55,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215350) roll requested 2024-11-24T20:46:55,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215360 2024-11-24T20:46:55,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215360 2024-11-24T20:46:55,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215360) roll requested 2024-11-24T20:46:55,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215371 2024-11-24T20:46:55,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215371 2024-11-24T20:46:55,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215371) roll requested 2024-11-24T20:46:55,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215380 2024-11-24T20:46:55,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215380 2024-11-24T20:46:55,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215380) roll requested 2024-11-24T20:46:55,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215387 2024-11-24T20:46:55,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215387 2024-11-24T20:46:55,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215387) roll requested 2024-11-24T20:46:55,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215397 2024-11-24T20:46:55,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215397 2024-11-24T20:46:55,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215397) roll requested 2024-11-24T20:46:55,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215403 2024-11-24T20:46:55,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215403 2024-11-24T20:46:55,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215403) roll requested 2024-11-24T20:46:55,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215413 2024-11-24T20:46:55,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215413 2024-11-24T20:46:55,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215413) roll requested 2024-11-24T20:46:55,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215419 2024-11-24T20:46:55,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215419 2024-11-24T20:46:55,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215419) roll requested 2024-11-24T20:46:55,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215428 2024-11-24T20:46:55,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215428 2024-11-24T20:46:55,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215428) roll requested 2024-11-24T20:46:55,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215440 2024-11-24T20:46:55,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215440 2024-11-24T20:46:55,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215440) roll requested 2024-11-24T20:46:55,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215453 2024-11-24T20:46:55,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215453 2024-11-24T20:46:55,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215453) roll requested 2024-11-24T20:46:55,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215460 2024-11-24T20:46:55,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215460 2024-11-24T20:46:55,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215460) roll requested 2024-11-24T20:46:55,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215469 2024-11-24T20:46:55,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215469 2024-11-24T20:46:55,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215469) roll requested 2024-11-24T20:46:55,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215476 2024-11-24T20:46:55,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215476 2024-11-24T20:46:55,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215476) roll requested 2024-11-24T20:46:55,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215484 2024-11-24T20:46:55,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215484 2024-11-24T20:46:55,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215484) roll requested 2024-11-24T20:46:55,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215492 2024-11-24T20:46:55,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215492 2024-11-24T20:46:55,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215492) roll requested 2024-11-24T20:46:55,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215499 2024-11-24T20:46:55,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215499 2024-11-24T20:46:55,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215499) roll requested 2024-11-24T20:46:55,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215514 2024-11-24T20:46:55,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215514 2024-11-24T20:46:55,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215514) roll requested 2024-11-24T20:46:55,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215520 2024-11-24T20:46:55,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215520 2024-11-24T20:46:55,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215520) roll requested 2024-11-24T20:46:55,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215527 2024-11-24T20:46:55,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215527 2024-11-24T20:46:55,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215527) roll requested 2024-11-24T20:46:55,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215534 2024-11-24T20:46:55,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215534 2024-11-24T20:46:55,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215534) roll requested 2024-11-24T20:46:55,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215545 2024-11-24T20:46:55,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215545 2024-11-24T20:46:55,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215545) roll requested 2024-11-24T20:46:55,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215553 2024-11-24T20:46:55,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215553 2024-11-24T20:46:55,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215553) roll requested 2024-11-24T20:46:55,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215561 2024-11-24T20:46:55,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215561 2024-11-24T20:46:55,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215561) roll requested 2024-11-24T20:46:55,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215569 2024-11-24T20:46:55,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215569 2024-11-24T20:46:55,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215569) roll requested 2024-11-24T20:46:55,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215577 2024-11-24T20:46:55,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215577 2024-11-24T20:46:55,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215577) roll requested 2024-11-24T20:46:55,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215586 2024-11-24T20:46:55,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215586 2024-11-24T20:46:55,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215586) roll requested 2024-11-24T20:46:55,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215593 2024-11-24T20:46:55,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215593 2024-11-24T20:46:55,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215593) roll requested 2024-11-24T20:46:55,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215601 2024-11-24T20:46:55,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215601 2024-11-24T20:46:55,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215601) roll requested 2024-11-24T20:46:55,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215608 2024-11-24T20:46:55,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215608 2024-11-24T20:46:55,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215608) roll requested 2024-11-24T20:46:55,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215615 2024-11-24T20:46:55,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215615 2024-11-24T20:46:55,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215615) roll requested 2024-11-24T20:46:55,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215627 2024-11-24T20:46:55,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215627 2024-11-24T20:46:55,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215627) roll requested 2024-11-24T20:46:55,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215634 2024-11-24T20:46:55,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215634 2024-11-24T20:46:55,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215634) roll requested 2024-11-24T20:46:55,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215642 2024-11-24T20:46:55,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215642 2024-11-24T20:46:55,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215642) roll requested 2024-11-24T20:46:55,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215652 2024-11-24T20:46:55,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215652 2024-11-24T20:46:55,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215652) roll requested 2024-11-24T20:46:55,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215659 2024-11-24T20:46:55,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215659 2024-11-24T20:46:55,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215659) roll requested 2024-11-24T20:46:55,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215669 2024-11-24T20:46:55,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215669 2024-11-24T20:46:55,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215669) roll requested 2024-11-24T20:46:55,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215676 2024-11-24T20:46:55,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215676 2024-11-24T20:46:55,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215676) roll requested 2024-11-24T20:46:55,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215684 2024-11-24T20:46:55,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215684 2024-11-24T20:46:55,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215684) roll requested 2024-11-24T20:46:55,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215692 2024-11-24T20:46:55,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215692 2024-11-24T20:46:55,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215692) roll requested 2024-11-24T20:46:55,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215699 2024-11-24T20:46:55,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215699 2024-11-24T20:46:55,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215699) roll requested 2024-11-24T20:46:55,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215707 2024-11-24T20:46:55,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215707 2024-11-24T20:46:55,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215707) roll requested 2024-11-24T20:46:55,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215719 2024-11-24T20:46:55,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215719 2024-11-24T20:46:55,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215719) roll requested 2024-11-24T20:46:55,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215728 2024-11-24T20:46:55,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215728 2024-11-24T20:46:55,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215728) roll requested 2024-11-24T20:46:55,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215736 2024-11-24T20:46:55,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215736 2024-11-24T20:46:55,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215736) roll requested 2024-11-24T20:46:55,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215745 2024-11-24T20:46:55,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215745 2024-11-24T20:46:55,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215745) roll requested 2024-11-24T20:46:55,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215752 2024-11-24T20:46:55,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215752 2024-11-24T20:46:55,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215752) roll requested 2024-11-24T20:46:55,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215758 2024-11-24T20:46:55,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215758 2024-11-24T20:46:55,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215758) roll requested 2024-11-24T20:46:55,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215765 2024-11-24T20:46:55,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215765 2024-11-24T20:46:55,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215765) roll requested 2024-11-24T20:46:55,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215776 2024-11-24T20:46:55,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215776 2024-11-24T20:46:55,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215776) roll requested 2024-11-24T20:46:55,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215784 2024-11-24T20:46:55,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215784 2024-11-24T20:46:55,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215784) roll requested 2024-11-24T20:46:55,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215791 2024-11-24T20:46:55,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215791 2024-11-24T20:46:55,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215791) roll requested 2024-11-24T20:46:55,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215797 2024-11-24T20:46:55,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215797 2024-11-24T20:46:55,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215797) roll requested 2024-11-24T20:46:55,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215804 2024-11-24T20:46:55,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215804 2024-11-24T20:46:55,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215804) roll requested 2024-11-24T20:46:55,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215810 2024-11-24T20:46:55,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215810 2024-11-24T20:46:55,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215810) roll requested 2024-11-24T20:46:55,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215817 2024-11-24T20:46:55,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215817 2024-11-24T20:46:55,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215817) roll requested 2024-11-24T20:46:55,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215831 2024-11-24T20:46:55,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215831 2024-11-24T20:46:55,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215831) roll requested 2024-11-24T20:46:55,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215838 2024-11-24T20:46:55,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215838 2024-11-24T20:46:55,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215838) roll requested 2024-11-24T20:46:55,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215848 2024-11-24T20:46:55,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215848 2024-11-24T20:46:55,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215848) roll requested 2024-11-24T20:46:55,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215854 2024-11-24T20:46:55,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215854 2024-11-24T20:46:55,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215854) roll requested 2024-11-24T20:46:55,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215861 2024-11-24T20:46:55,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215861 2024-11-24T20:46:55,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215861) roll requested 2024-11-24T20:46:55,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215868 2024-11-24T20:46:55,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215868 2024-11-24T20:46:55,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215868) roll requested 2024-11-24T20:46:55,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215874 2024-11-24T20:46:55,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215874 2024-11-24T20:46:55,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215874) roll requested 2024-11-24T20:46:55,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215881 2024-11-24T20:46:55,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215881 2024-11-24T20:46:55,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215881) roll requested 2024-11-24T20:46:55,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215887 2024-11-24T20:46:55,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215887 2024-11-24T20:46:55,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215887) roll requested 2024-11-24T20:46:55,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215897 2024-11-24T20:46:55,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215897 2024-11-24T20:46:55,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215897) roll requested 2024-11-24T20:46:55,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215905 2024-11-24T20:46:55,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215905 2024-11-24T20:46:55,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215905) roll requested 2024-11-24T20:46:55,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215916 2024-11-24T20:46:55,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215916 2024-11-24T20:46:55,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215916) roll requested 2024-11-24T20:46:55,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215925 2024-11-24T20:46:55,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215925 2024-11-24T20:46:55,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215925) roll requested 2024-11-24T20:46:55,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215936 2024-11-24T20:46:55,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215936 2024-11-24T20:46:55,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215936) roll requested 2024-11-24T20:46:55,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215946 2024-11-24T20:46:55,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215946 2024-11-24T20:46:55,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215946) roll requested 2024-11-24T20:46:55,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215957 2024-11-24T20:46:55,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215957 2024-11-24T20:46:55,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215957) roll requested 2024-11-24T20:46:55,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215967 2024-11-24T20:46:55,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215967 2024-11-24T20:46:55,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215967) roll requested 2024-11-24T20:46:55,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215975 2024-11-24T20:46:55,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215975 2024-11-24T20:46:55,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215975) roll requested 2024-11-24T20:46:55,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215984 2024-11-24T20:46:55,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215984 2024-11-24T20:46:55,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215984) roll requested 2024-11-24T20:46:55,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215993 2024-11-24T20:46:55,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:55,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215993 2024-11-24T20:46:55,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:55,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:55,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215993) roll requested 2024-11-24T20:46:55,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481215999 2024-11-24T20:46:56,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215999 2024-11-24T20:46:56,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481215999) roll requested 2024-11-24T20:46:56,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216006 2024-11-24T20:46:56,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481215999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216006 2024-11-24T20:46:56,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216006) roll requested 2024-11-24T20:46:56,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216012 2024-11-24T20:46:56,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216012 2024-11-24T20:46:56,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216012) roll requested 2024-11-24T20:46:56,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216018 2024-11-24T20:46:56,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216018 2024-11-24T20:46:56,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216018) roll requested 2024-11-24T20:46:56,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216025 2024-11-24T20:46:56,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216025 2024-11-24T20:46:56,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216025) roll requested 2024-11-24T20:46:56,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216034 2024-11-24T20:46:56,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216034 2024-11-24T20:46:56,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216034) roll requested 2024-11-24T20:46:56,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216043 2024-11-24T20:46:56,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216043 2024-11-24T20:46:56,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216043) roll requested 2024-11-24T20:46:56,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216052 2024-11-24T20:46:56,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216052 2024-11-24T20:46:56,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216052) roll requested 2024-11-24T20:46:56,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216061 2024-11-24T20:46:56,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216061 2024-11-24T20:46:56,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216061) roll requested 2024-11-24T20:46:56,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216070 2024-11-24T20:46:56,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216070 2024-11-24T20:46:56,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216070) roll requested 2024-11-24T20:46:56,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216079 2024-11-24T20:46:56,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216079 2024-11-24T20:46:56,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216079) roll requested 2024-11-24T20:46:56,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216089 2024-11-24T20:46:56,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216089 2024-11-24T20:46:56,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216089) roll requested 2024-11-24T20:46:56,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216100 2024-11-24T20:46:56,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216100 2024-11-24T20:46:56,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216100) roll requested 2024-11-24T20:46:56,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216107 2024-11-24T20:46:56,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216107 2024-11-24T20:46:56,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216107) roll requested 2024-11-24T20:46:56,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216116 2024-11-24T20:46:56,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216116 2024-11-24T20:46:56,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216116) roll requested 2024-11-24T20:46:56,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216123 2024-11-24T20:46:56,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216123 2024-11-24T20:46:56,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216123) roll requested 2024-11-24T20:46:56,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216129 2024-11-24T20:46:56,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216129 2024-11-24T20:46:56,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216129) roll requested 2024-11-24T20:46:56,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216135 2024-11-24T20:46:56,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216135 2024-11-24T20:46:56,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216135) roll requested 2024-11-24T20:46:56,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216143 2024-11-24T20:46:56,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216143 2024-11-24T20:46:56,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216143) roll requested 2024-11-24T20:46:56,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216149 2024-11-24T20:46:56,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216149 2024-11-24T20:46:56,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216149) roll requested 2024-11-24T20:46:56,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216156 2024-11-24T20:46:56,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216156 2024-11-24T20:46:56,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216156) roll requested 2024-11-24T20:46:56,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216165 2024-11-24T20:46:56,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216165 2024-11-24T20:46:56,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216165) roll requested 2024-11-24T20:46:56,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216171 2024-11-24T20:46:56,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216171 2024-11-24T20:46:56,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216171) roll requested 2024-11-24T20:46:56,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216179 2024-11-24T20:46:56,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216179 2024-11-24T20:46:56,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216179) roll requested 2024-11-24T20:46:56,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216186 2024-11-24T20:46:56,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216186 2024-11-24T20:46:56,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216186) roll requested 2024-11-24T20:46:56,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216192 2024-11-24T20:46:56,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216192 2024-11-24T20:46:56,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216192) roll requested 2024-11-24T20:46:56,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216202 2024-11-24T20:46:56,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216202 2024-11-24T20:46:56,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216202) roll requested 2024-11-24T20:46:56,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216213 2024-11-24T20:46:56,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216213 2024-11-24T20:46:56,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216213) roll requested 2024-11-24T20:46:56,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216219 2024-11-24T20:46:56,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216219 2024-11-24T20:46:56,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216219) roll requested 2024-11-24T20:46:56,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216226 2024-11-24T20:46:56,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216226 2024-11-24T20:46:56,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216226) roll requested 2024-11-24T20:46:56,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216237 2024-11-24T20:46:56,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216237 2024-11-24T20:46:56,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216237) roll requested 2024-11-24T20:46:56,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216246 2024-11-24T20:46:56,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216246 2024-11-24T20:46:56,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216246) roll requested 2024-11-24T20:46:56,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216262 2024-11-24T20:46:56,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216262 2024-11-24T20:46:56,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216262) roll requested 2024-11-24T20:46:56,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216273 2024-11-24T20:46:56,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216273 2024-11-24T20:46:56,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216273) roll requested 2024-11-24T20:46:56,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216280 2024-11-24T20:46:56,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216280 2024-11-24T20:46:56,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216280) roll requested 2024-11-24T20:46:56,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216289 2024-11-24T20:46:56,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216289 2024-11-24T20:46:56,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216289) roll requested 2024-11-24T20:46:56,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216300 2024-11-24T20:46:56,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216300 2024-11-24T20:46:56,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216300) roll requested 2024-11-24T20:46:56,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216308 2024-11-24T20:46:56,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216308 2024-11-24T20:46:56,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216308) roll requested 2024-11-24T20:46:56,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216323 2024-11-24T20:46:56,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216323 2024-11-24T20:46:56,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216323) roll requested 2024-11-24T20:46:56,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216332 2024-11-24T20:46:56,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216332 2024-11-24T20:46:56,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216332) roll requested 2024-11-24T20:46:56,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216340 2024-11-24T20:46:56,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216340 2024-11-24T20:46:56,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216340) roll requested 2024-11-24T20:46:56,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216352 2024-11-24T20:46:56,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216352 2024-11-24T20:46:56,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216352) roll requested 2024-11-24T20:46:56,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216363 2024-11-24T20:46:56,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216363 2024-11-24T20:46:56,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216363) roll requested 2024-11-24T20:46:56,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216371 2024-11-24T20:46:56,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216371 2024-11-24T20:46:56,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216371) roll requested 2024-11-24T20:46:56,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216380 2024-11-24T20:46:56,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216380 2024-11-24T20:46:56,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216380) roll requested 2024-11-24T20:46:56,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216386 2024-11-24T20:46:56,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216386 2024-11-24T20:46:56,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216386) roll requested 2024-11-24T20:46:56,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216393 2024-11-24T20:46:56,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216393 2024-11-24T20:46:56,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216393) roll requested 2024-11-24T20:46:56,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216402 2024-11-24T20:46:56,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216402 2024-11-24T20:46:56,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216402) roll requested 2024-11-24T20:46:56,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216412 2024-11-24T20:46:56,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216412 2024-11-24T20:46:56,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216412) roll requested 2024-11-24T20:46:56,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216419 2024-11-24T20:46:56,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216419 2024-11-24T20:46:56,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216419) roll requested 2024-11-24T20:46:56,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216426 2024-11-24T20:46:56,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216426 2024-11-24T20:46:56,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216426) roll requested 2024-11-24T20:46:56,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216432 2024-11-24T20:46:56,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216432 2024-11-24T20:46:56,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216432) roll requested 2024-11-24T20:46:56,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216439 2024-11-24T20:46:56,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216439 2024-11-24T20:46:56,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216439) roll requested 2024-11-24T20:46:56,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216445 2024-11-24T20:46:56,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216445 2024-11-24T20:46:56,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216445) roll requested 2024-11-24T20:46:56,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216451 2024-11-24T20:46:56,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216451 2024-11-24T20:46:56,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216451) roll requested 2024-11-24T20:46:56,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216456 2024-11-24T20:46:56,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216456 2024-11-24T20:46:56,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216456) roll requested 2024-11-24T20:46:56,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216466 2024-11-24T20:46:56,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216466 2024-11-24T20:46:56,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216466) roll requested 2024-11-24T20:46:56,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216472 2024-11-24T20:46:56,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216472 2024-11-24T20:46:56,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216472) roll requested 2024-11-24T20:46:56,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216478 2024-11-24T20:46:56,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216478 2024-11-24T20:46:56,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216478) roll requested 2024-11-24T20:46:56,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216488 2024-11-24T20:46:56,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216488 2024-11-24T20:46:56,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216488) roll requested 2024-11-24T20:46:56,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216498 2024-11-24T20:46:56,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216498 2024-11-24T20:46:56,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216498) roll requested 2024-11-24T20:46:56,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216506 2024-11-24T20:46:56,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216506 2024-11-24T20:46:56,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216506) roll requested 2024-11-24T20:46:56,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216512 2024-11-24T20:46:56,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216512 2024-11-24T20:46:56,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216512) roll requested 2024-11-24T20:46:56,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216522 2024-11-24T20:46:56,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216522 2024-11-24T20:46:56,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216522) roll requested 2024-11-24T20:46:56,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216528 2024-11-24T20:46:56,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216528 2024-11-24T20:46:56,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216528) roll requested 2024-11-24T20:46:56,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216536 2024-11-24T20:46:56,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216536 2024-11-24T20:46:56,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216536) roll requested 2024-11-24T20:46:56,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216545 2024-11-24T20:46:56,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216545 2024-11-24T20:46:56,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216545) roll requested 2024-11-24T20:46:56,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216553 2024-11-24T20:46:56,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216553 2024-11-24T20:46:56,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216553) roll requested 2024-11-24T20:46:56,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216562 2024-11-24T20:46:56,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216562 2024-11-24T20:46:56,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216562) roll requested 2024-11-24T20:46:56,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216572 2024-11-24T20:46:56,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216572 2024-11-24T20:46:56,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216572) roll requested 2024-11-24T20:46:56,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216584 2024-11-24T20:46:56,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216584 2024-11-24T20:46:56,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216584) roll requested 2024-11-24T20:46:56,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216591 2024-11-24T20:46:56,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216591 2024-11-24T20:46:56,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216591) roll requested 2024-11-24T20:46:56,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216597 2024-11-24T20:46:56,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216597 2024-11-24T20:46:56,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216597) roll requested 2024-11-24T20:46:56,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216605 2024-11-24T20:46:56,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216605 2024-11-24T20:46:56,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216605) roll requested 2024-11-24T20:46:56,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216611 2024-11-24T20:46:56,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216611 2024-11-24T20:46:56,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216611) roll requested 2024-11-24T20:46:56,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216620 2024-11-24T20:46:56,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216620 2024-11-24T20:46:56,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216620) roll requested 2024-11-24T20:46:56,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216627 2024-11-24T20:46:56,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216627 2024-11-24T20:46:56,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216627) roll requested 2024-11-24T20:46:56,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216633 2024-11-24T20:46:56,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216633 2024-11-24T20:46:56,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216633) roll requested 2024-11-24T20:46:56,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216644 2024-11-24T20:46:56,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216644 2024-11-24T20:46:56,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216644) roll requested 2024-11-24T20:46:56,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216651 2024-11-24T20:46:56,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216651 2024-11-24T20:46:56,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216651) roll requested 2024-11-24T20:46:56,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216665 2024-11-24T20:46:56,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216665 2024-11-24T20:46:56,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216665) roll requested 2024-11-24T20:46:56,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216674 2024-11-24T20:46:56,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216674 2024-11-24T20:46:56,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216674) roll requested 2024-11-24T20:46:56,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216689 2024-11-24T20:46:56,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216689 2024-11-24T20:46:56,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216689) roll requested 2024-11-24T20:46:56,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216698 2024-11-24T20:46:56,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216698 2024-11-24T20:46:56,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216698) roll requested 2024-11-24T20:46:56,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216705 2024-11-24T20:46:56,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216705 2024-11-24T20:46:56,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216705) roll requested 2024-11-24T20:46:56,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216711 2024-11-24T20:46:56,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216711 2024-11-24T20:46:56,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216711) roll requested 2024-11-24T20:46:56,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216718 2024-11-24T20:46:56,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216718 2024-11-24T20:46:56,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216718) roll requested 2024-11-24T20:46:56,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216727 2024-11-24T20:46:56,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216727 2024-11-24T20:46:56,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216727) roll requested 2024-11-24T20:46:56,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216737 2024-11-24T20:46:56,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216737 2024-11-24T20:46:56,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216737) roll requested 2024-11-24T20:46:56,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216743 2024-11-24T20:46:56,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216743 2024-11-24T20:46:56,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216743) roll requested 2024-11-24T20:46:56,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216750 2024-11-24T20:46:56,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216750 2024-11-24T20:46:56,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216750) roll requested 2024-11-24T20:46:56,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216759 2024-11-24T20:46:56,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216759 2024-11-24T20:46:56,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216759) roll requested 2024-11-24T20:46:56,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216766 2024-11-24T20:46:56,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216766 2024-11-24T20:46:56,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216766) roll requested 2024-11-24T20:46:56,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216773 2024-11-24T20:46:56,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216773 2024-11-24T20:46:56,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216773) roll requested 2024-11-24T20:46:56,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216779 2024-11-24T20:46:56,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216779 2024-11-24T20:46:56,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216779) roll requested 2024-11-24T20:46:56,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216786 2024-11-24T20:46:56,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216786 2024-11-24T20:46:56,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216786) roll requested 2024-11-24T20:46:56,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216792 2024-11-24T20:46:56,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216792 2024-11-24T20:46:56,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216792) roll requested 2024-11-24T20:46:56,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216798 2024-11-24T20:46:56,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216798 2024-11-24T20:46:56,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216798) roll requested 2024-11-24T20:46:56,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216805 2024-11-24T20:46:56,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216805 2024-11-24T20:46:56,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216805) roll requested 2024-11-24T20:46:56,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216811 2024-11-24T20:46:56,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216811 2024-11-24T20:46:56,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216811) roll requested 2024-11-24T20:46:56,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216818 2024-11-24T20:46:56,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216818 2024-11-24T20:46:56,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216818) roll requested 2024-11-24T20:46:56,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216823 2024-11-24T20:46:56,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216823 2024-11-24T20:46:56,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216823) roll requested 2024-11-24T20:46:56,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216829 2024-11-24T20:46:56,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216829 2024-11-24T20:46:56,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216829) roll requested 2024-11-24T20:46:56,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216835 2024-11-24T20:46:56,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216835 2024-11-24T20:46:56,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216835) roll requested 2024-11-24T20:46:56,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216842 2024-11-24T20:46:56,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216842 2024-11-24T20:46:56,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216842) roll requested 2024-11-24T20:46:56,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216852 2024-11-24T20:46:56,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216852 2024-11-24T20:46:56,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216852) roll requested 2024-11-24T20:46:56,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216861 2024-11-24T20:46:56,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216861 2024-11-24T20:46:56,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216861) roll requested 2024-11-24T20:46:56,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216868 2024-11-24T20:46:56,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216868 2024-11-24T20:46:56,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216868) roll requested 2024-11-24T20:46:56,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216876 2024-11-24T20:46:56,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216876 2024-11-24T20:46:56,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216876) roll requested 2024-11-24T20:46:56,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216883 2024-11-24T20:46:56,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216883 2024-11-24T20:46:56,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216876 is not closed yet, will try archiving it next time 2024-11-24T20:46:56,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216883) roll requested 2024-11-24T20:46:56,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216889 2024-11-24T20:46:56,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216889 2024-11-24T20:46:56,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216889) roll requested 2024-11-24T20:46:56,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216894 2024-11-24T20:46:56,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216894 2024-11-24T20:46:56,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216894) roll requested 2024-11-24T20:46:56,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216900 2024-11-24T20:46:56,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216900 2024-11-24T20:46:56,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216900) roll requested 2024-11-24T20:46:56,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216906 2024-11-24T20:46:56,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216906 2024-11-24T20:46:56,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216906) roll requested 2024-11-24T20:46:56,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216915 2024-11-24T20:46:56,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216915 2024-11-24T20:46:56,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216915) roll requested 2024-11-24T20:46:56,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216923 2024-11-24T20:46:56,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216923 2024-11-24T20:46:56,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216923) roll requested 2024-11-24T20:46:56,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216934 2024-11-24T20:46:56,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216934 2024-11-24T20:46:56,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216934) roll requested 2024-11-24T20:46:56,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216940 2024-11-24T20:46:56,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216940 2024-11-24T20:46:56,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216940) roll requested 2024-11-24T20:46:56,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216947 2024-11-24T20:46:56,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216947 2024-11-24T20:46:56,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216947) roll requested 2024-11-24T20:46:56,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216962 2024-11-24T20:46:56,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216962 2024-11-24T20:46:56,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216962) roll requested 2024-11-24T20:46:56,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216967 2024-11-24T20:46:56,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216967 2024-11-24T20:46:56,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216967) roll requested 2024-11-24T20:46:56,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216973 2024-11-24T20:46:56,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216973 2024-11-24T20:46:56,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216973) roll requested 2024-11-24T20:46:56,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216981 2024-11-24T20:46:56,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216981 2024-11-24T20:46:56,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216981) roll requested 2024-11-24T20:46:56,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216989 2024-11-24T20:46:56,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:56,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216989 2024-11-24T20:46:56,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:56,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:56,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216989) roll requested 2024-11-24T20:46:56,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481216998 2024-11-24T20:46:57,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216998 2024-11-24T20:46:57,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481216998) roll requested 2024-11-24T20:46:57,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217009 2024-11-24T20:46:57,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481216998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217009 2024-11-24T20:46:57,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217009) roll requested 2024-11-24T20:46:57,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217019 2024-11-24T20:46:57,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217019 2024-11-24T20:46:57,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217019) roll requested 2024-11-24T20:46:57,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217029 2024-11-24T20:46:57,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217029 2024-11-24T20:46:57,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217029) roll requested 2024-11-24T20:46:57,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217039 2024-11-24T20:46:57,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217039 2024-11-24T20:46:57,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217039) roll requested 2024-11-24T20:46:57,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217056 2024-11-24T20:46:57,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217056 2024-11-24T20:46:57,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217056) roll requested 2024-11-24T20:46:57,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217063 2024-11-24T20:46:57,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217063 2024-11-24T20:46:57,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217063) roll requested 2024-11-24T20:46:57,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217074 2024-11-24T20:46:57,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217074 2024-11-24T20:46:57,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217074) roll requested 2024-11-24T20:46:57,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217080 2024-11-24T20:46:57,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217080 2024-11-24T20:46:57,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217080) roll requested 2024-11-24T20:46:57,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217086 2024-11-24T20:46:57,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217086 2024-11-24T20:46:57,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217086) roll requested 2024-11-24T20:46:57,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217092 2024-11-24T20:46:57,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217092 2024-11-24T20:46:57,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217092) roll requested 2024-11-24T20:46:57,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217100 2024-11-24T20:46:57,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217100 2024-11-24T20:46:57,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217100) roll requested 2024-11-24T20:46:57,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217107 2024-11-24T20:46:57,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217107 2024-11-24T20:46:57,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217107) roll requested 2024-11-24T20:46:57,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217116 2024-11-24T20:46:57,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217116 2024-11-24T20:46:57,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217116) roll requested 2024-11-24T20:46:57,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217124 2024-11-24T20:46:57,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217124 2024-11-24T20:46:57,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217124) roll requested 2024-11-24T20:46:57,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217134 2024-11-24T20:46:57,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217134 2024-11-24T20:46:57,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217134) roll requested 2024-11-24T20:46:57,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217140 2024-11-24T20:46:57,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217140 2024-11-24T20:46:57,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217140) roll requested 2024-11-24T20:46:57,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217148 2024-11-24T20:46:57,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217148 2024-11-24T20:46:57,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217148) roll requested 2024-11-24T20:46:57,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217156 2024-11-24T20:46:57,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217156 2024-11-24T20:46:57,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217156) roll requested 2024-11-24T20:46:57,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217165 2024-11-24T20:46:57,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217165 2024-11-24T20:46:57,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217165) roll requested 2024-11-24T20:46:57,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217174 2024-11-24T20:46:57,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217174 2024-11-24T20:46:57,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217174) roll requested 2024-11-24T20:46:57,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217187 2024-11-24T20:46:57,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217187 2024-11-24T20:46:57,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217187) roll requested 2024-11-24T20:46:57,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217198 2024-11-24T20:46:57,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217198 2024-11-24T20:46:57,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217198) roll requested 2024-11-24T20:46:57,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217208 2024-11-24T20:46:57,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217208 2024-11-24T20:46:57,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217208) roll requested 2024-11-24T20:46:57,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217214 2024-11-24T20:46:57,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217214 2024-11-24T20:46:57,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217214) roll requested 2024-11-24T20:46:57,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217220 2024-11-24T20:46:57,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217220 2024-11-24T20:46:57,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217220) roll requested 2024-11-24T20:46:57,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217226 2024-11-24T20:46:57,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217226 2024-11-24T20:46:57,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217226) roll requested 2024-11-24T20:46:57,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217232 2024-11-24T20:46:57,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217232 2024-11-24T20:46:57,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217232) roll requested 2024-11-24T20:46:57,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217239 2024-11-24T20:46:57,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217239 2024-11-24T20:46:57,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217239) roll requested 2024-11-24T20:46:57,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217247 2024-11-24T20:46:57,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217247 2024-11-24T20:46:57,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217247) roll requested 2024-11-24T20:46:57,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217256 2024-11-24T20:46:57,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217256 2024-11-24T20:46:57,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217256) roll requested 2024-11-24T20:46:57,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217268 2024-11-24T20:46:57,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217268 2024-11-24T20:46:57,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217268) roll requested 2024-11-24T20:46:57,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217276 2024-11-24T20:46:57,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217276 2024-11-24T20:46:57,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217276) roll requested 2024-11-24T20:46:57,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217282 2024-11-24T20:46:57,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217282 2024-11-24T20:46:57,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217282) roll requested 2024-11-24T20:46:57,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217291 2024-11-24T20:46:57,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217291 2024-11-24T20:46:57,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217291) roll requested 2024-11-24T20:46:57,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217297 2024-11-24T20:46:57,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217297 2024-11-24T20:46:57,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217297) roll requested 2024-11-24T20:46:57,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217303 2024-11-24T20:46:57,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217303 2024-11-24T20:46:57,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217303) roll requested 2024-11-24T20:46:57,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217309 2024-11-24T20:46:57,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217309 2024-11-24T20:46:57,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217309) roll requested 2024-11-24T20:46:57,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217318 2024-11-24T20:46:57,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217318 2024-11-24T20:46:57,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217318) roll requested 2024-11-24T20:46:57,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217327 2024-11-24T20:46:57,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217327 2024-11-24T20:46:57,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217327) roll requested 2024-11-24T20:46:57,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217336 2024-11-24T20:46:57,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217336 2024-11-24T20:46:57,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217336) roll requested 2024-11-24T20:46:57,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217342 2024-11-24T20:46:57,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217342 2024-11-24T20:46:57,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217342) roll requested 2024-11-24T20:46:57,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217350 2024-11-24T20:46:57,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217350 2024-11-24T20:46:57,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217350) roll requested 2024-11-24T20:46:57,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217357 2024-11-24T20:46:57,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217357 2024-11-24T20:46:57,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217357) roll requested 2024-11-24T20:46:57,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217368 2024-11-24T20:46:57,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217368 2024-11-24T20:46:57,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217368) roll requested 2024-11-24T20:46:57,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217375 2024-11-24T20:46:57,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217375 2024-11-24T20:46:57,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217375) roll requested 2024-11-24T20:46:57,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217382 2024-11-24T20:46:57,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217382 2024-11-24T20:46:57,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217382) roll requested 2024-11-24T20:46:57,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217388 2024-11-24T20:46:57,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217388 2024-11-24T20:46:57,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217388) roll requested 2024-11-24T20:46:57,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217394 2024-11-24T20:46:57,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217394 2024-11-24T20:46:57,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217394) roll requested 2024-11-24T20:46:57,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217404 2024-11-24T20:46:57,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217404 2024-11-24T20:46:57,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217404) roll requested 2024-11-24T20:46:57,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217412 2024-11-24T20:46:57,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217412 2024-11-24T20:46:57,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217412) roll requested 2024-11-24T20:46:57,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217418 2024-11-24T20:46:57,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217418 2024-11-24T20:46:57,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217418) roll requested 2024-11-24T20:46:57,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217424 2024-11-24T20:46:57,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217424 2024-11-24T20:46:57,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217424) roll requested 2024-11-24T20:46:57,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217431 2024-11-24T20:46:57,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217431 2024-11-24T20:46:57,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217431) roll requested 2024-11-24T20:46:57,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217437 2024-11-24T20:46:57,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217437 2024-11-24T20:46:57,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217437) roll requested 2024-11-24T20:46:57,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217447 2024-11-24T20:46:57,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217447 2024-11-24T20:46:57,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217447) roll requested 2024-11-24T20:46:57,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217457 2024-11-24T20:46:57,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217457 2024-11-24T20:46:57,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217457) roll requested 2024-11-24T20:46:57,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217468 2024-11-24T20:46:57,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217468 2024-11-24T20:46:57,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217468) roll requested 2024-11-24T20:46:57,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217477 2024-11-24T20:46:57,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217477 2024-11-24T20:46:57,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217477) roll requested 2024-11-24T20:46:57,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217483 2024-11-24T20:46:57,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217483 2024-11-24T20:46:57,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217483) roll requested 2024-11-24T20:46:57,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217490 2024-11-24T20:46:57,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217490 2024-11-24T20:46:57,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217490) roll requested 2024-11-24T20:46:57,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217499 2024-11-24T20:46:57,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217499 2024-11-24T20:46:57,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217499) roll requested 2024-11-24T20:46:57,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217508 2024-11-24T20:46:57,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217508 2024-11-24T20:46:57,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217508) roll requested 2024-11-24T20:46:57,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217517 2024-11-24T20:46:57,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217517 2024-11-24T20:46:57,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217517) roll requested 2024-11-24T20:46:57,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217525 2024-11-24T20:46:57,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217525 2024-11-24T20:46:57,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217525) roll requested 2024-11-24T20:46:57,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217541 2024-11-24T20:46:57,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217541 2024-11-24T20:46:57,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217541) roll requested 2024-11-24T20:46:57,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217547 2024-11-24T20:46:57,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217547 2024-11-24T20:46:57,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217547) roll requested 2024-11-24T20:46:57,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217552 2024-11-24T20:46:57,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217552 2024-11-24T20:46:57,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217552) roll requested 2024-11-24T20:46:57,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217559 2024-11-24T20:46:57,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217559 2024-11-24T20:46:57,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217559) roll requested 2024-11-24T20:46:57,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217568 2024-11-24T20:46:57,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217568 2024-11-24T20:46:57,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217568) roll requested 2024-11-24T20:46:57,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217576 2024-11-24T20:46:57,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217576 2024-11-24T20:46:57,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217576) roll requested 2024-11-24T20:46:57,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217585 2024-11-24T20:46:57,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217585 2024-11-24T20:46:57,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217585) roll requested 2024-11-24T20:46:57,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217596 2024-11-24T20:46:57,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217596 2024-11-24T20:46:57,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217596) roll requested 2024-11-24T20:46:57,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217602 2024-11-24T20:46:57,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217602 2024-11-24T20:46:57,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217602) roll requested 2024-11-24T20:46:57,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217607 2024-11-24T20:46:57,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217607 2024-11-24T20:46:57,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217607) roll requested 2024-11-24T20:46:57,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217614 2024-11-24T20:46:57,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217614 2024-11-24T20:46:57,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217614) roll requested 2024-11-24T20:46:57,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217622 2024-11-24T20:46:57,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217622 2024-11-24T20:46:57,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217622) roll requested 2024-11-24T20:46:57,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217628 2024-11-24T20:46:57,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217628 2024-11-24T20:46:57,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217628) roll requested 2024-11-24T20:46:57,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217634 2024-11-24T20:46:57,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217634 2024-11-24T20:46:57,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217634) roll requested 2024-11-24T20:46:57,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217640 2024-11-24T20:46:57,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217640 2024-11-24T20:46:57,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217640) roll requested 2024-11-24T20:46:57,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217647 2024-11-24T20:46:57,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217647 2024-11-24T20:46:57,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217647) roll requested 2024-11-24T20:46:57,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217653 2024-11-24T20:46:57,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217653 2024-11-24T20:46:57,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217653) roll requested 2024-11-24T20:46:57,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217661 2024-11-24T20:46:57,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217661 2024-11-24T20:46:57,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217661) roll requested 2024-11-24T20:46:57,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217670 2024-11-24T20:46:57,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217670 2024-11-24T20:46:57,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217670) roll requested 2024-11-24T20:46:57,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217680 2024-11-24T20:46:57,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217680 2024-11-24T20:46:57,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217680) roll requested 2024-11-24T20:46:57,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217689 2024-11-24T20:46:57,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217689 2024-11-24T20:46:57,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217689) roll requested 2024-11-24T20:46:57,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217698 2024-11-24T20:46:57,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217698 2024-11-24T20:46:57,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217698) roll requested 2024-11-24T20:46:57,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217708 2024-11-24T20:46:57,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217708 2024-11-24T20:46:57,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217708) roll requested 2024-11-24T20:46:57,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217716 2024-11-24T20:46:57,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217716 2024-11-24T20:46:57,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217716) roll requested 2024-11-24T20:46:57,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217722 2024-11-24T20:46:57,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217722 2024-11-24T20:46:57,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217722) roll requested 2024-11-24T20:46:57,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217728 2024-11-24T20:46:57,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217728 2024-11-24T20:46:57,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217728) roll requested 2024-11-24T20:46:57,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217738 2024-11-24T20:46:57,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217738 2024-11-24T20:46:57,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217738) roll requested 2024-11-24T20:46:57,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217744 2024-11-24T20:46:57,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217744 2024-11-24T20:46:57,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217744) roll requested 2024-11-24T20:46:57,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217749 2024-11-24T20:46:57,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217749 2024-11-24T20:46:57,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217749) roll requested 2024-11-24T20:46:57,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217756 2024-11-24T20:46:57,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217756 2024-11-24T20:46:57,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217756) roll requested 2024-11-24T20:46:57,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217763 2024-11-24T20:46:57,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217763 2024-11-24T20:46:57,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217763) roll requested 2024-11-24T20:46:57,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217772 2024-11-24T20:46:57,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217772 2024-11-24T20:46:57,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217772) roll requested 2024-11-24T20:46:57,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217782 2024-11-24T20:46:57,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217782 2024-11-24T20:46:57,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217782) roll requested 2024-11-24T20:46:57,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217790 2024-11-24T20:46:57,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217790 2024-11-24T20:46:57,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217790) roll requested 2024-11-24T20:46:57,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217801 2024-11-24T20:46:57,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217801 2024-11-24T20:46:57,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217801) roll requested 2024-11-24T20:46:57,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217808 2024-11-24T20:46:57,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217808 2024-11-24T20:46:57,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217808) roll requested 2024-11-24T20:46:57,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217815 2024-11-24T20:46:57,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217815 2024-11-24T20:46:57,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217815) roll requested 2024-11-24T20:46:57,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217822 2024-11-24T20:46:57,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217822 2024-11-24T20:46:57,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217822) roll requested 2024-11-24T20:46:57,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217829 2024-11-24T20:46:57,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217829 2024-11-24T20:46:57,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217829) roll requested 2024-11-24T20:46:57,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217841 2024-11-24T20:46:57,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217841 2024-11-24T20:46:57,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217841) roll requested 2024-11-24T20:46:57,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217849 2024-11-24T20:46:57,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217849 2024-11-24T20:46:57,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217849) roll requested 2024-11-24T20:46:57,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217856 2024-11-24T20:46:57,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217856 2024-11-24T20:46:57,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217856) roll requested 2024-11-24T20:46:57,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217871 2024-11-24T20:46:57,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217871 2024-11-24T20:46:57,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217871) roll requested 2024-11-24T20:46:57,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217881 2024-11-24T20:46:57,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217881 2024-11-24T20:46:57,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217881) roll requested 2024-11-24T20:46:57,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217898 2024-11-24T20:46:57,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217898 2024-11-24T20:46:57,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217898) roll requested 2024-11-24T20:46:57,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217908 2024-11-24T20:46:57,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217908 2024-11-24T20:46:57,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217908) roll requested 2024-11-24T20:46:57,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217916 2024-11-24T20:46:57,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217916 2024-11-24T20:46:57,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217916) roll requested 2024-11-24T20:46:57,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217925 2024-11-24T20:46:57,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217925 2024-11-24T20:46:57,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217925) roll requested 2024-11-24T20:46:57,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217932 2024-11-24T20:46:57,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217932 2024-11-24T20:46:57,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217932) roll requested 2024-11-24T20:46:57,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217940 2024-11-24T20:46:57,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217940 2024-11-24T20:46:57,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217940) roll requested 2024-11-24T20:46:57,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217949 2024-11-24T20:46:57,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217949 2024-11-24T20:46:57,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217949) roll requested 2024-11-24T20:46:57,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217958 2024-11-24T20:46:57,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217958 2024-11-24T20:46:57,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217958) roll requested 2024-11-24T20:46:57,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217967 2024-11-24T20:46:57,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217967 2024-11-24T20:46:57,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217967) roll requested 2024-11-24T20:46:57,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217981 2024-11-24T20:46:57,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217981 2024-11-24T20:46:57,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217981) roll requested 2024-11-24T20:46:57,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217989 2024-11-24T20:46:57,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:57,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217989 2024-11-24T20:46:57,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:57,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:57,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217989) roll requested 2024-11-24T20:46:57,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481217996 2024-11-24T20:46:58,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217996 2024-11-24T20:46:58,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481217996) roll requested 2024-11-24T20:46:58,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218008 2024-11-24T20:46:58,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481217996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218008 2024-11-24T20:46:58,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218008) roll requested 2024-11-24T20:46:58,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218016 2024-11-24T20:46:58,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218016 2024-11-24T20:46:58,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218016) roll requested 2024-11-24T20:46:58,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218022 2024-11-24T20:46:58,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218022 2024-11-24T20:46:58,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218022) roll requested 2024-11-24T20:46:58,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218028 2024-11-24T20:46:58,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218028 2024-11-24T20:46:58,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218028) roll requested 2024-11-24T20:46:58,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218036 2024-11-24T20:46:58,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218036 2024-11-24T20:46:58,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218036) roll requested 2024-11-24T20:46:58,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218042 2024-11-24T20:46:58,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218042 2024-11-24T20:46:58,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218042) roll requested 2024-11-24T20:46:58,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218048 2024-11-24T20:46:58,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218048 2024-11-24T20:46:58,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218048) roll requested 2024-11-24T20:46:58,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218058 2024-11-24T20:46:58,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218058 2024-11-24T20:46:58,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218058) roll requested 2024-11-24T20:46:58,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218068 2024-11-24T20:46:58,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218068 2024-11-24T20:46:58,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218068) roll requested 2024-11-24T20:46:58,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218075 2024-11-24T20:46:58,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218075 2024-11-24T20:46:58,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218075) roll requested 2024-11-24T20:46:58,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218081 2024-11-24T20:46:58,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218081 2024-11-24T20:46:58,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218081) roll requested 2024-11-24T20:46:58,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218087 2024-11-24T20:46:58,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218087 2024-11-24T20:46:58,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218087) roll requested 2024-11-24T20:46:58,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218093 2024-11-24T20:46:58,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218093 2024-11-24T20:46:58,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218093) roll requested 2024-11-24T20:46:58,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218099 2024-11-24T20:46:58,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218099 2024-11-24T20:46:58,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218099) roll requested 2024-11-24T20:46:58,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218105 2024-11-24T20:46:58,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218105 2024-11-24T20:46:58,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218105) roll requested 2024-11-24T20:46:58,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218111 2024-11-24T20:46:58,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218111 2024-11-24T20:46:58,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218111) roll requested 2024-11-24T20:46:58,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218117 2024-11-24T20:46:58,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218117 2024-11-24T20:46:58,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218117) roll requested 2024-11-24T20:46:58,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218122 2024-11-24T20:46:58,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218122 2024-11-24T20:46:58,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218122) roll requested 2024-11-24T20:46:58,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218128 2024-11-24T20:46:58,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218128 2024-11-24T20:46:58,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218128) roll requested 2024-11-24T20:46:58,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218134 2024-11-24T20:46:58,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218134 2024-11-24T20:46:58,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218134) roll requested 2024-11-24T20:46:58,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218140 2024-11-24T20:46:58,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218140 2024-11-24T20:46:58,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218140) roll requested 2024-11-24T20:46:58,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218146 2024-11-24T20:46:58,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218146 2024-11-24T20:46:58,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218146) roll requested 2024-11-24T20:46:58,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218154 2024-11-24T20:46:58,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218154 2024-11-24T20:46:58,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218154) roll requested 2024-11-24T20:46:58,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218160 2024-11-24T20:46:58,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218160 2024-11-24T20:46:58,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218160) roll requested 2024-11-24T20:46:58,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218165 2024-11-24T20:46:58,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218165 2024-11-24T20:46:58,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218165) roll requested 2024-11-24T20:46:58,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218171 2024-11-24T20:46:58,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218171 2024-11-24T20:46:58,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218171) roll requested 2024-11-24T20:46:58,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218176 2024-11-24T20:46:58,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218176 2024-11-24T20:46:58,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218176) roll requested 2024-11-24T20:46:58,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218182 2024-11-24T20:46:58,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218182 2024-11-24T20:46:58,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218182) roll requested 2024-11-24T20:46:58,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218188 2024-11-24T20:46:58,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218188 2024-11-24T20:46:58,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218188) roll requested 2024-11-24T20:46:58,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218194 2024-11-24T20:46:58,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218194 2024-11-24T20:46:58,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218194) roll requested 2024-11-24T20:46:58,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218199 2024-11-24T20:46:58,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218199 2024-11-24T20:46:58,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218199) roll requested 2024-11-24T20:46:58,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218204 2024-11-24T20:46:58,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218204 2024-11-24T20:46:58,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218204) roll requested 2024-11-24T20:46:58,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218209 2024-11-24T20:46:58,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218209 2024-11-24T20:46:58,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218209) roll requested 2024-11-24T20:46:58,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218215 2024-11-24T20:46:58,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218215 2024-11-24T20:46:58,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218215) roll requested 2024-11-24T20:46:58,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218226 2024-11-24T20:46:58,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218226 2024-11-24T20:46:58,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218226) roll requested 2024-11-24T20:46:58,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218234 2024-11-24T20:46:58,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218234 2024-11-24T20:46:58,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218234) roll requested 2024-11-24T20:46:58,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218243 2024-11-24T20:46:58,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218243 2024-11-24T20:46:58,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218243) roll requested 2024-11-24T20:46:58,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218251 2024-11-24T20:46:58,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218251 2024-11-24T20:46:58,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218251) roll requested 2024-11-24T20:46:58,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218260 2024-11-24T20:46:58,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218260 2024-11-24T20:46:58,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218260) roll requested 2024-11-24T20:46:58,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218266 2024-11-24T20:46:58,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218266 2024-11-24T20:46:58,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218266) roll requested 2024-11-24T20:46:58,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218276 2024-11-24T20:46:58,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218276 2024-11-24T20:46:58,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218276) roll requested 2024-11-24T20:46:58,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218284 2024-11-24T20:46:58,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218284 2024-11-24T20:46:58,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218284) roll requested 2024-11-24T20:46:58,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218289 2024-11-24T20:46:58,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218289 2024-11-24T20:46:58,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218289) roll requested 2024-11-24T20:46:58,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218295 2024-11-24T20:46:58,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218295 2024-11-24T20:46:58,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218295) roll requested 2024-11-24T20:46:58,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218304 2024-11-24T20:46:58,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218304 2024-11-24T20:46:58,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218304) roll requested 2024-11-24T20:46:58,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218312 2024-11-24T20:46:58,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218312 2024-11-24T20:46:58,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218312) roll requested 2024-11-24T20:46:58,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218318 2024-11-24T20:46:58,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218318 2024-11-24T20:46:58,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218318) roll requested 2024-11-24T20:46:58,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218328 2024-11-24T20:46:58,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218328 2024-11-24T20:46:58,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218328) roll requested 2024-11-24T20:46:58,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218334 2024-11-24T20:46:58,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218334 2024-11-24T20:46:58,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218334) roll requested 2024-11-24T20:46:58,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218344 2024-11-24T20:46:58,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218344 2024-11-24T20:46:58,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218344) roll requested 2024-11-24T20:46:58,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218351 2024-11-24T20:46:58,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218351 2024-11-24T20:46:58,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218351) roll requested 2024-11-24T20:46:58,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218356 2024-11-24T20:46:58,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218356 2024-11-24T20:46:58,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218356) roll requested 2024-11-24T20:46:58,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218362 2024-11-24T20:46:58,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218362 2024-11-24T20:46:58,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218362) roll requested 2024-11-24T20:46:58,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218369 2024-11-24T20:46:58,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218369 2024-11-24T20:46:58,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218369) roll requested 2024-11-24T20:46:58,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218376 2024-11-24T20:46:58,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218376 2024-11-24T20:46:58,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218376) roll requested 2024-11-24T20:46:58,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218384 2024-11-24T20:46:58,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218384 2024-11-24T20:46:58,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218384) roll requested 2024-11-24T20:46:58,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218391 2024-11-24T20:46:58,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218391 2024-11-24T20:46:58,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218391) roll requested 2024-11-24T20:46:58,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218399 2024-11-24T20:46:58,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218399 2024-11-24T20:46:58,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218399) roll requested 2024-11-24T20:46:58,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218408 2024-11-24T20:46:58,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218408 2024-11-24T20:46:58,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218408) roll requested 2024-11-24T20:46:58,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218417 2024-11-24T20:46:58,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218417 2024-11-24T20:46:58,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218417) roll requested 2024-11-24T20:46:58,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218428 2024-11-24T20:46:58,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218428 2024-11-24T20:46:58,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218428) roll requested 2024-11-24T20:46:58,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218437 2024-11-24T20:46:58,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218437 2024-11-24T20:46:58,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218437) roll requested 2024-11-24T20:46:58,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218445 2024-11-24T20:46:58,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218445 2024-11-24T20:46:58,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218445) roll requested 2024-11-24T20:46:58,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218455 2024-11-24T20:46:58,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218455 2024-11-24T20:46:58,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218455) roll requested 2024-11-24T20:46:58,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218465 2024-11-24T20:46:58,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218465 2024-11-24T20:46:58,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218465) roll requested 2024-11-24T20:46:58,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218475 2024-11-24T20:46:58,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218475 2024-11-24T20:46:58,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218475) roll requested 2024-11-24T20:46:58,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218481 2024-11-24T20:46:58,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218481 2024-11-24T20:46:58,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218481) roll requested 2024-11-24T20:46:58,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218489 2024-11-24T20:46:58,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218489 2024-11-24T20:46:58,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218489) roll requested 2024-11-24T20:46:58,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218496 2024-11-24T20:46:58,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218496 2024-11-24T20:46:58,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218496) roll requested 2024-11-24T20:46:58,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218503 2024-11-24T20:46:58,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218503 2024-11-24T20:46:58,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218503) roll requested 2024-11-24T20:46:58,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218513 2024-11-24T20:46:58,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218513 2024-11-24T20:46:58,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218513) roll requested 2024-11-24T20:46:58,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218522 2024-11-24T20:46:58,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218522 2024-11-24T20:46:58,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218522) roll requested 2024-11-24T20:46:58,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218531 2024-11-24T20:46:58,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218531 2024-11-24T20:46:58,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218531) roll requested 2024-11-24T20:46:58,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218537 2024-11-24T20:46:58,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218537 2024-11-24T20:46:58,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218537) roll requested 2024-11-24T20:46:58,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218544 2024-11-24T20:46:58,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218544 2024-11-24T20:46:58,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218544) roll requested 2024-11-24T20:46:58,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218559 2024-11-24T20:46:58,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218559 2024-11-24T20:46:58,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218559) roll requested 2024-11-24T20:46:58,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218573 2024-11-24T20:46:58,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218573 2024-11-24T20:46:58,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218573) roll requested 2024-11-24T20:46:58,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218581 2024-11-24T20:46:58,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218581 2024-11-24T20:46:58,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218581) roll requested 2024-11-24T20:46:58,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218589 2024-11-24T20:46:58,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218589 2024-11-24T20:46:58,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218589) roll requested 2024-11-24T20:46:58,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218596 2024-11-24T20:46:58,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218596 2024-11-24T20:46:58,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218596) roll requested 2024-11-24T20:46:58,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218603 2024-11-24T20:46:58,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218603 2024-11-24T20:46:58,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218603) roll requested 2024-11-24T20:46:58,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218610 2024-11-24T20:46:58,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218610 2024-11-24T20:46:58,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218610) roll requested 2024-11-24T20:46:58,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218617 2024-11-24T20:46:58,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218617 2024-11-24T20:46:58,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218617) roll requested 2024-11-24T20:46:58,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218624 2024-11-24T20:46:58,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218624 2024-11-24T20:46:58,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218624) roll requested 2024-11-24T20:46:58,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218632 2024-11-24T20:46:58,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218632 2024-11-24T20:46:58,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218632) roll requested 2024-11-24T20:46:58,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218648 2024-11-24T20:46:58,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218648 2024-11-24T20:46:58,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218648) roll requested 2024-11-24T20:46:58,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218656 2024-11-24T20:46:58,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218656 2024-11-24T20:46:58,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218656) roll requested 2024-11-24T20:46:58,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218669 2024-11-24T20:46:58,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218669 2024-11-24T20:46:58,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218669) roll requested 2024-11-24T20:46:58,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218676 2024-11-24T20:46:58,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218676 2024-11-24T20:46:58,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218676) roll requested 2024-11-24T20:46:58,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218684 2024-11-24T20:46:58,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218684 2024-11-24T20:46:58,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218684) roll requested 2024-11-24T20:46:58,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218690 2024-11-24T20:46:58,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218690 2024-11-24T20:46:58,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218690) roll requested 2024-11-24T20:46:58,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218695 2024-11-24T20:46:58,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218695 2024-11-24T20:46:58,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218695) roll requested 2024-11-24T20:46:58,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218700 2024-11-24T20:46:58,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218700 2024-11-24T20:46:58,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218700) roll requested 2024-11-24T20:46:58,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218707 2024-11-24T20:46:58,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218707 2024-11-24T20:46:58,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218707) roll requested 2024-11-24T20:46:58,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218714 2024-11-24T20:46:58,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218714 2024-11-24T20:46:58,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218714) roll requested 2024-11-24T20:46:58,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218721 2024-11-24T20:46:58,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218721 2024-11-24T20:46:58,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218721) roll requested 2024-11-24T20:46:58,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218730 2024-11-24T20:46:58,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218730 2024-11-24T20:46:58,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218730) roll requested 2024-11-24T20:46:58,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218737 2024-11-24T20:46:58,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218737 2024-11-24T20:46:58,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218737) roll requested 2024-11-24T20:46:58,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218746 2024-11-24T20:46:58,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218746 2024-11-24T20:46:58,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218746) roll requested 2024-11-24T20:46:58,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218755 2024-11-24T20:46:58,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218755 2024-11-24T20:46:58,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218755) roll requested 2024-11-24T20:46:58,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218765 2024-11-24T20:46:58,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218765 2024-11-24T20:46:58,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218765) roll requested 2024-11-24T20:46:58,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218774 2024-11-24T20:46:58,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218774 2024-11-24T20:46:58,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218774) roll requested 2024-11-24T20:46:58,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218780 2024-11-24T20:46:58,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218780 2024-11-24T20:46:58,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218780) roll requested 2024-11-24T20:46:58,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218785 2024-11-24T20:46:58,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218785 2024-11-24T20:46:58,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218785) roll requested 2024-11-24T20:46:58,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218789 2024-11-24T20:46:58,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218789 2024-11-24T20:46:58,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218789) roll requested 2024-11-24T20:46:58,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218795 2024-11-24T20:46:58,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218795 2024-11-24T20:46:58,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218795) roll requested 2024-11-24T20:46:58,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218801 2024-11-24T20:46:58,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218801 2024-11-24T20:46:58,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218801) roll requested 2024-11-24T20:46:58,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218808 2024-11-24T20:46:58,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218808 2024-11-24T20:46:58,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218808) roll requested 2024-11-24T20:46:58,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218816 2024-11-24T20:46:58,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218816 2024-11-24T20:46:58,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218816) roll requested 2024-11-24T20:46:58,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218826 2024-11-24T20:46:58,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218826 2024-11-24T20:46:58,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218826) roll requested 2024-11-24T20:46:58,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218832 2024-11-24T20:46:58,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218832 2024-11-24T20:46:58,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218832) roll requested 2024-11-24T20:46:58,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218837 2024-11-24T20:46:58,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218837 2024-11-24T20:46:58,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218837) roll requested 2024-11-24T20:46:58,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218844 2024-11-24T20:46:58,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218844 2024-11-24T20:46:58,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218844) roll requested 2024-11-24T20:46:58,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218849 2024-11-24T20:46:58,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218849 2024-11-24T20:46:58,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218849) roll requested 2024-11-24T20:46:58,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218854 2024-11-24T20:46:58,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218854 2024-11-24T20:46:58,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218854) roll requested 2024-11-24T20:46:58,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218859 2024-11-24T20:46:58,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218859 2024-11-24T20:46:58,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218859) roll requested 2024-11-24T20:46:58,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218865 2024-11-24T20:46:58,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218865 2024-11-24T20:46:58,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218865) roll requested 2024-11-24T20:46:58,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218873 2024-11-24T20:46:58,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218873 2024-11-24T20:46:58,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218873) roll requested 2024-11-24T20:46:58,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218880 2024-11-24T20:46:58,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218880 2024-11-24T20:46:58,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218880) roll requested 2024-11-24T20:46:58,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218891 2024-11-24T20:46:58,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218891 2024-11-24T20:46:58,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218891) roll requested 2024-11-24T20:46:58,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218897 2024-11-24T20:46:58,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218897 2024-11-24T20:46:58,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218897) roll requested 2024-11-24T20:46:58,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218904 2024-11-24T20:46:58,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218904 2024-11-24T20:46:58,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218904) roll requested 2024-11-24T20:46:58,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218912 2024-11-24T20:46:58,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218912 2024-11-24T20:46:58,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218912) roll requested 2024-11-24T20:46:58,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218920 2024-11-24T20:46:58,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218920 2024-11-24T20:46:58,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218920) roll requested 2024-11-24T20:46:58,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218926 2024-11-24T20:46:58,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218926 2024-11-24T20:46:58,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218926) roll requested 2024-11-24T20:46:58,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218933 2024-11-24T20:46:58,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218933 2024-11-24T20:46:58,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218933) roll requested 2024-11-24T20:46:58,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218941 2024-11-24T20:46:58,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218941 2024-11-24T20:46:58,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218941) roll requested 2024-11-24T20:46:58,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218949 2024-11-24T20:46:58,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218949 2024-11-24T20:46:58,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218949) roll requested 2024-11-24T20:46:58,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218960 2024-11-24T20:46:58,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218960 2024-11-24T20:46:58,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218960) roll requested 2024-11-24T20:46:58,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218965 2024-11-24T20:46:58,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218965 2024-11-24T20:46:58,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218965) roll requested 2024-11-24T20:46:58,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218971 2024-11-24T20:46:58,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218971 2024-11-24T20:46:58,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218971) roll requested 2024-11-24T20:46:58,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218977 2024-11-24T20:46:58,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218977 2024-11-24T20:46:58,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218977) roll requested 2024-11-24T20:46:58,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218984 2024-11-24T20:46:58,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:58,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218984 2024-11-24T20:46:58,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:58,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:58,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218984) roll requested 2024-11-24T20:46:58,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481218993 2024-11-24T20:46:59,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218993 2024-11-24T20:46:59,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481218993) roll requested 2024-11-24T20:46:59,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219006 2024-11-24T20:46:59,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481218993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219006 2024-11-24T20:46:59,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219006) roll requested 2024-11-24T20:46:59,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219012 2024-11-24T20:46:59,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219012 2024-11-24T20:46:59,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219012) roll requested 2024-11-24T20:46:59,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219020 2024-11-24T20:46:59,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219020 2024-11-24T20:46:59,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219020) roll requested 2024-11-24T20:46:59,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219026 2024-11-24T20:46:59,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219026 2024-11-24T20:46:59,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219026) roll requested 2024-11-24T20:46:59,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219033 2024-11-24T20:46:59,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219033 2024-11-24T20:46:59,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219033) roll requested 2024-11-24T20:46:59,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219041 2024-11-24T20:46:59,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219041 2024-11-24T20:46:59,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219041) roll requested 2024-11-24T20:46:59,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219050 2024-11-24T20:46:59,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219050 2024-11-24T20:46:59,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219050) roll requested 2024-11-24T20:46:59,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219061 2024-11-24T20:46:59,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219061 2024-11-24T20:46:59,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219061) roll requested 2024-11-24T20:46:59,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219071 2024-11-24T20:46:59,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219071 2024-11-24T20:46:59,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219071) roll requested 2024-11-24T20:46:59,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219076 2024-11-24T20:46:59,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219076 2024-11-24T20:46:59,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219076) roll requested 2024-11-24T20:46:59,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219082 2024-11-24T20:46:59,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219082 2024-11-24T20:46:59,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219082) roll requested 2024-11-24T20:46:59,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219089 2024-11-24T20:46:59,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219089 2024-11-24T20:46:59,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219089) roll requested 2024-11-24T20:46:59,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219096 2024-11-24T20:46:59,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219096 2024-11-24T20:46:59,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219096) roll requested 2024-11-24T20:46:59,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219104 2024-11-24T20:46:59,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219104 2024-11-24T20:46:59,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219104) roll requested 2024-11-24T20:46:59,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219112 2024-11-24T20:46:59,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219112 2024-11-24T20:46:59,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219112) roll requested 2024-11-24T20:46:59,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219120 2024-11-24T20:46:59,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219120 2024-11-24T20:46:59,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219120) roll requested 2024-11-24T20:46:59,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219129 2024-11-24T20:46:59,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219129 2024-11-24T20:46:59,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219129) roll requested 2024-11-24T20:46:59,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219134 2024-11-24T20:46:59,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219134 2024-11-24T20:46:59,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219134) roll requested 2024-11-24T20:46:59,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219140 2024-11-24T20:46:59,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219140 2024-11-24T20:46:59,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219140) roll requested 2024-11-24T20:46:59,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219146 2024-11-24T20:46:59,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219146 2024-11-24T20:46:59,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219146) roll requested 2024-11-24T20:46:59,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219151 2024-11-24T20:46:59,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219151 2024-11-24T20:46:59,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219151) roll requested 2024-11-24T20:46:59,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219158 2024-11-24T20:46:59,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219158 2024-11-24T20:46:59,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219158) roll requested 2024-11-24T20:46:59,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219165 2024-11-24T20:46:59,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219165 2024-11-24T20:46:59,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219165) roll requested 2024-11-24T20:46:59,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219173 2024-11-24T20:46:59,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219173 2024-11-24T20:46:59,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219173) roll requested 2024-11-24T20:46:59,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219181 2024-11-24T20:46:59,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219181 2024-11-24T20:46:59,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219181) roll requested 2024-11-24T20:46:59,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219188 2024-11-24T20:46:59,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219188 2024-11-24T20:46:59,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219188) roll requested 2024-11-24T20:46:59,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219196 2024-11-24T20:46:59,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219196 2024-11-24T20:46:59,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219196) roll requested 2024-11-24T20:46:59,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219208 2024-11-24T20:46:59,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219208 2024-11-24T20:46:59,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219208) roll requested 2024-11-24T20:46:59,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219215 2024-11-24T20:46:59,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219215 2024-11-24T20:46:59,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219215) roll requested 2024-11-24T20:46:59,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219220 2024-11-24T20:46:59,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219220 2024-11-24T20:46:59,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219220) roll requested 2024-11-24T20:46:59,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219225 2024-11-24T20:46:59,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219225 2024-11-24T20:46:59,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219225) roll requested 2024-11-24T20:46:59,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219231 2024-11-24T20:46:59,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219231 2024-11-24T20:46:59,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219231) roll requested 2024-11-24T20:46:59,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219239 2024-11-24T20:46:59,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219239 2024-11-24T20:46:59,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219239) roll requested 2024-11-24T20:46:59,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219246 2024-11-24T20:46:59,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219246 2024-11-24T20:46:59,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219246) roll requested 2024-11-24T20:46:59,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219251 2024-11-24T20:46:59,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219251 2024-11-24T20:46:59,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219251) roll requested 2024-11-24T20:46:59,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219256 2024-11-24T20:46:59,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219256 2024-11-24T20:46:59,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219256) roll requested 2024-11-24T20:46:59,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219262 2024-11-24T20:46:59,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219262 2024-11-24T20:46:59,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219262) roll requested 2024-11-24T20:46:59,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219270 2024-11-24T20:46:59,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219270 2024-11-24T20:46:59,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219270) roll requested 2024-11-24T20:46:59,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219277 2024-11-24T20:46:59,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219277 2024-11-24T20:46:59,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219277) roll requested 2024-11-24T20:46:59,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219284 2024-11-24T20:46:59,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219284 2024-11-24T20:46:59,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219284) roll requested 2024-11-24T20:46:59,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219294 2024-11-24T20:46:59,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219294 2024-11-24T20:46:59,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219294) roll requested 2024-11-24T20:46:59,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219304 2024-11-24T20:46:59,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219304 2024-11-24T20:46:59,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219304) roll requested 2024-11-24T20:46:59,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219312 2024-11-24T20:46:59,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219312 2024-11-24T20:46:59,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219312) roll requested 2024-11-24T20:46:59,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219320 2024-11-24T20:46:59,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219320 2024-11-24T20:46:59,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219320) roll requested 2024-11-24T20:46:59,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219327 2024-11-24T20:46:59,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219327 2024-11-24T20:46:59,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219327) roll requested 2024-11-24T20:46:59,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219335 2024-11-24T20:46:59,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219335 2024-11-24T20:46:59,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219335) roll requested 2024-11-24T20:46:59,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219343 2024-11-24T20:46:59,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219343 2024-11-24T20:46:59,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219343) roll requested 2024-11-24T20:46:59,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219352 2024-11-24T20:46:59,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219352 2024-11-24T20:46:59,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219352) roll requested 2024-11-24T20:46:59,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219360 2024-11-24T20:46:59,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219360 2024-11-24T20:46:59,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219360) roll requested 2024-11-24T20:46:59,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219368 2024-11-24T20:46:59,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219368 2024-11-24T20:46:59,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219368) roll requested 2024-11-24T20:46:59,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219376 2024-11-24T20:46:59,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219376 2024-11-24T20:46:59,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219376) roll requested 2024-11-24T20:46:59,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219384 2024-11-24T20:46:59,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219384 2024-11-24T20:46:59,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219384) roll requested 2024-11-24T20:46:59,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219393 2024-11-24T20:46:59,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219393 2024-11-24T20:46:59,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219393) roll requested 2024-11-24T20:46:59,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219400 2024-11-24T20:46:59,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219400 2024-11-24T20:46:59,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219400) roll requested 2024-11-24T20:46:59,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219406 2024-11-24T20:46:59,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219406 2024-11-24T20:46:59,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219406) roll requested 2024-11-24T20:46:59,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219412 2024-11-24T20:46:59,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219412 2024-11-24T20:46:59,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219412) roll requested 2024-11-24T20:46:59,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219420 2024-11-24T20:46:59,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219420 2024-11-24T20:46:59,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219420) roll requested 2024-11-24T20:46:59,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219427 2024-11-24T20:46:59,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219427 2024-11-24T20:46:59,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219427) roll requested 2024-11-24T20:46:59,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219437 2024-11-24T20:46:59,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219437 2024-11-24T20:46:59,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219437) roll requested 2024-11-24T20:46:59,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219446 2024-11-24T20:46:59,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219446 2024-11-24T20:46:59,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219446) roll requested 2024-11-24T20:46:59,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219454 2024-11-24T20:46:59,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219454 2024-11-24T20:46:59,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219454) roll requested 2024-11-24T20:46:59,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219463 2024-11-24T20:46:59,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219463 2024-11-24T20:46:59,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219463) roll requested 2024-11-24T20:46:59,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219473 2024-11-24T20:46:59,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219473 2024-11-24T20:46:59,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219473) roll requested 2024-11-24T20:46:59,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219483 2024-11-24T20:46:59,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219483 2024-11-24T20:46:59,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219483) roll requested 2024-11-24T20:46:59,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219492 2024-11-24T20:46:59,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219492 2024-11-24T20:46:59,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219492) roll requested 2024-11-24T20:46:59,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219500 2024-11-24T20:46:59,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219500 2024-11-24T20:46:59,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219500) roll requested 2024-11-24T20:46:59,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219507 2024-11-24T20:46:59,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219507 2024-11-24T20:46:59,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219507) roll requested 2024-11-24T20:46:59,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219519 2024-11-24T20:46:59,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219519 2024-11-24T20:46:59,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219519) roll requested 2024-11-24T20:46:59,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219525 2024-11-24T20:46:59,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219525 2024-11-24T20:46:59,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219525) roll requested 2024-11-24T20:46:59,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219530 2024-11-24T20:46:59,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219530 2024-11-24T20:46:59,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219530) roll requested 2024-11-24T20:46:59,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219535 2024-11-24T20:46:59,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219535 2024-11-24T20:46:59,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219535) roll requested 2024-11-24T20:46:59,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219541 2024-11-24T20:46:59,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219541 2024-11-24T20:46:59,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219541) roll requested 2024-11-24T20:46:59,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219548 2024-11-24T20:46:59,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219548 2024-11-24T20:46:59,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219548) roll requested 2024-11-24T20:46:59,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219556 2024-11-24T20:46:59,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219556 2024-11-24T20:46:59,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219556) roll requested 2024-11-24T20:46:59,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219564 2024-11-24T20:46:59,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219564 2024-11-24T20:46:59,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219564) roll requested 2024-11-24T20:46:59,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219572 2024-11-24T20:46:59,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219572 2024-11-24T20:46:59,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219572) roll requested 2024-11-24T20:46:59,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219579 2024-11-24T20:46:59,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219579 2024-11-24T20:46:59,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219579) roll requested 2024-11-24T20:46:59,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219586 2024-11-24T20:46:59,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219586 2024-11-24T20:46:59,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219586) roll requested 2024-11-24T20:46:59,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219594 2024-11-24T20:46:59,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219594 2024-11-24T20:46:59,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219594) roll requested 2024-11-24T20:46:59,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219603 2024-11-24T20:46:59,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219603 2024-11-24T20:46:59,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219603) roll requested 2024-11-24T20:46:59,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219612 2024-11-24T20:46:59,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219612 2024-11-24T20:46:59,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219612) roll requested 2024-11-24T20:46:59,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219619 2024-11-24T20:46:59,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219619 2024-11-24T20:46:59,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219619) roll requested 2024-11-24T20:46:59,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219626 2024-11-24T20:46:59,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219626 2024-11-24T20:46:59,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219626) roll requested 2024-11-24T20:46:59,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219634 2024-11-24T20:46:59,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219634 2024-11-24T20:46:59,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219634) roll requested 2024-11-24T20:46:59,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219644 2024-11-24T20:46:59,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219644 2024-11-24T20:46:59,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219644) roll requested 2024-11-24T20:46:59,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219650 2024-11-24T20:46:59,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219650 2024-11-24T20:46:59,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219650) roll requested 2024-11-24T20:46:59,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219656 2024-11-24T20:46:59,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219656 2024-11-24T20:46:59,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219656) roll requested 2024-11-24T20:46:59,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219668 2024-11-24T20:46:59,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219668 2024-11-24T20:46:59,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219668) roll requested 2024-11-24T20:46:59,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219674 2024-11-24T20:46:59,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219674 2024-11-24T20:46:59,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219674) roll requested 2024-11-24T20:46:59,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219680 2024-11-24T20:46:59,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219680 2024-11-24T20:46:59,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219680) roll requested 2024-11-24T20:46:59,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219686 2024-11-24T20:46:59,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219686 2024-11-24T20:46:59,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219686) roll requested 2024-11-24T20:46:59,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219694 2024-11-24T20:46:59,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219694 2024-11-24T20:46:59,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219694) roll requested 2024-11-24T20:46:59,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219701 2024-11-24T20:46:59,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219701 2024-11-24T20:46:59,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219701) roll requested 2024-11-24T20:46:59,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219710 2024-11-24T20:46:59,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219710 2024-11-24T20:46:59,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219710) roll requested 2024-11-24T20:46:59,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219718 2024-11-24T20:46:59,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219718 2024-11-24T20:46:59,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219718) roll requested 2024-11-24T20:46:59,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219726 2024-11-24T20:46:59,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219726 2024-11-24T20:46:59,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219726) roll requested 2024-11-24T20:46:59,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219736 2024-11-24T20:46:59,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219736 2024-11-24T20:46:59,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219736) roll requested 2024-11-24T20:46:59,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219746 2024-11-24T20:46:59,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219746 2024-11-24T20:46:59,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219746) roll requested 2024-11-24T20:46:59,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219752 2024-11-24T20:46:59,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219752 2024-11-24T20:46:59,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219752) roll requested 2024-11-24T20:46:59,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219760 2024-11-24T20:46:59,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219760 2024-11-24T20:46:59,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219760) roll requested 2024-11-24T20:46:59,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219770 2024-11-24T20:46:59,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219770 2024-11-24T20:46:59,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219770) roll requested 2024-11-24T20:46:59,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219779 2024-11-24T20:46:59,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219779 2024-11-24T20:46:59,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219779) roll requested 2024-11-24T20:46:59,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219788 2024-11-24T20:46:59,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219788 2024-11-24T20:46:59,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219788) roll requested 2024-11-24T20:46:59,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219800 2024-11-24T20:46:59,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219800 2024-11-24T20:46:59,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219800) roll requested 2024-11-24T20:46:59,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219811 2024-11-24T20:46:59,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219811 2024-11-24T20:46:59,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219811) roll requested 2024-11-24T20:46:59,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219820 2024-11-24T20:46:59,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219820 2024-11-24T20:46:59,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219820) roll requested 2024-11-24T20:46:59,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219832 2024-11-24T20:46:59,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219832 2024-11-24T20:46:59,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219832) roll requested 2024-11-24T20:46:59,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219838 2024-11-24T20:46:59,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219838 2024-11-24T20:46:59,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219838) roll requested 2024-11-24T20:46:59,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219844 2024-11-24T20:46:59,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219844 2024-11-24T20:46:59,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219844) roll requested 2024-11-24T20:46:59,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219850 2024-11-24T20:46:59,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219850 2024-11-24T20:46:59,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219850) roll requested 2024-11-24T20:46:59,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219856 2024-11-24T20:46:59,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219856 2024-11-24T20:46:59,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219856) roll requested 2024-11-24T20:46:59,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219863 2024-11-24T20:46:59,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219863 2024-11-24T20:46:59,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219863) roll requested 2024-11-24T20:46:59,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219871 2024-11-24T20:46:59,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219871 2024-11-24T20:46:59,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219871) roll requested 2024-11-24T20:46:59,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219878 2024-11-24T20:46:59,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219878 2024-11-24T20:46:59,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219878) roll requested 2024-11-24T20:46:59,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219886 2024-11-24T20:46:59,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219886 2024-11-24T20:46:59,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219886) roll requested 2024-11-24T20:46:59,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219896 2024-11-24T20:46:59,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219896 2024-11-24T20:46:59,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219896) roll requested 2024-11-24T20:46:59,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219905 2024-11-24T20:46:59,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219905 2024-11-24T20:46:59,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219905) roll requested 2024-11-24T20:46:59,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219920 2024-11-24T20:46:59,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219920 2024-11-24T20:46:59,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219920) roll requested 2024-11-24T20:46:59,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219926 2024-11-24T20:46:59,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219926 2024-11-24T20:46:59,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219926) roll requested 2024-11-24T20:46:59,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219933 2024-11-24T20:46:59,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219933 2024-11-24T20:46:59,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219933) roll requested 2024-11-24T20:46:59,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219940 2024-11-24T20:46:59,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219940 2024-11-24T20:46:59,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219940) roll requested 2024-11-24T20:46:59,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219950 2024-11-24T20:46:59,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219950 2024-11-24T20:46:59,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219950) roll requested 2024-11-24T20:46:59,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219958 2024-11-24T20:46:59,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219958 2024-11-24T20:46:59,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219958) roll requested 2024-11-24T20:46:59,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219968 2024-11-24T20:46:59,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219968 2024-11-24T20:46:59,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219968) roll requested 2024-11-24T20:46:59,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219977 2024-11-24T20:46:59,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219977 2024-11-24T20:46:59,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219977) roll requested 2024-11-24T20:46:59,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219988 2024-11-24T20:46:59,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:46:59,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219988 2024-11-24T20:46:59,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:46:59,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:59,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219988) roll requested 2024-11-24T20:46:59,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481219998 2024-11-24T20:47:00,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219998 2024-11-24T20:47:00,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481219998) roll requested 2024-11-24T20:47:00,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220007 2024-11-24T20:47:00,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481219998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220007 2024-11-24T20:47:00,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220007) roll requested 2024-11-24T20:47:00,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220017 2024-11-24T20:47:00,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220017 2024-11-24T20:47:00,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220017) roll requested 2024-11-24T20:47:00,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220026 2024-11-24T20:47:00,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220026 2024-11-24T20:47:00,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220026) roll requested 2024-11-24T20:47:00,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220035 2024-11-24T20:47:00,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220035 2024-11-24T20:47:00,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220035) roll requested 2024-11-24T20:47:00,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220042 2024-11-24T20:47:00,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220042 2024-11-24T20:47:00,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220042) roll requested 2024-11-24T20:47:00,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220051 2024-11-24T20:47:00,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220051 2024-11-24T20:47:00,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220051) roll requested 2024-11-24T20:47:00,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220059 2024-11-24T20:47:00,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220059 2024-11-24T20:47:00,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220059) roll requested 2024-11-24T20:47:00,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220068 2024-11-24T20:47:00,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220068 2024-11-24T20:47:00,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220068) roll requested 2024-11-24T20:47:00,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220076 2024-11-24T20:47:00,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220076 2024-11-24T20:47:00,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220076) roll requested 2024-11-24T20:47:00,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220083 2024-11-24T20:47:00,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220083 2024-11-24T20:47:00,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220083) roll requested 2024-11-24T20:47:00,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220090 2024-11-24T20:47:00,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220090 2024-11-24T20:47:00,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220090) roll requested 2024-11-24T20:47:00,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220099 2024-11-24T20:47:00,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220099 2024-11-24T20:47:00,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220099) roll requested 2024-11-24T20:47:00,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220109 2024-11-24T20:47:00,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220109 2024-11-24T20:47:00,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220109) roll requested 2024-11-24T20:47:00,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220119 2024-11-24T20:47:00,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220119 2024-11-24T20:47:00,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220119) roll requested 2024-11-24T20:47:00,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220128 2024-11-24T20:47:00,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220128 2024-11-24T20:47:00,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220128) roll requested 2024-11-24T20:47:00,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220136 2024-11-24T20:47:00,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220136 2024-11-24T20:47:00,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220136) roll requested 2024-11-24T20:47:00,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220143 2024-11-24T20:47:00,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220143 2024-11-24T20:47:00,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220143) roll requested 2024-11-24T20:47:00,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220151 2024-11-24T20:47:00,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220151 2024-11-24T20:47:00,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220151) roll requested 2024-11-24T20:47:00,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220159 2024-11-24T20:47:00,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220159 2024-11-24T20:47:00,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220159) roll requested 2024-11-24T20:47:00,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220170 2024-11-24T20:47:00,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220170 2024-11-24T20:47:00,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220170) roll requested 2024-11-24T20:47:00,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220175 2024-11-24T20:47:00,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220175 2024-11-24T20:47:00,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220175) roll requested 2024-11-24T20:47:00,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220180 2024-11-24T20:47:00,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220180 2024-11-24T20:47:00,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220180) roll requested 2024-11-24T20:47:00,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220186 2024-11-24T20:47:00,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220186 2024-11-24T20:47:00,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220186) roll requested 2024-11-24T20:47:00,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220193 2024-11-24T20:47:00,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220193 2024-11-24T20:47:00,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220193) roll requested 2024-11-24T20:47:00,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220200 2024-11-24T20:47:00,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220200 2024-11-24T20:47:00,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220200) roll requested 2024-11-24T20:47:00,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220207 2024-11-24T20:47:00,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220207 2024-11-24T20:47:00,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220207) roll requested 2024-11-24T20:47:00,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220216 2024-11-24T20:47:00,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220216 2024-11-24T20:47:00,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220216) roll requested 2024-11-24T20:47:00,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220226 2024-11-24T20:47:00,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220226 2024-11-24T20:47:00,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220226) roll requested 2024-11-24T20:47:00,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220235 2024-11-24T20:47:00,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220235 2024-11-24T20:47:00,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220235) roll requested 2024-11-24T20:47:00,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220240 2024-11-24T20:47:00,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220240 2024-11-24T20:47:00,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220240) roll requested 2024-11-24T20:47:00,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220247 2024-11-24T20:47:00,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220247 2024-11-24T20:47:00,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220247) roll requested 2024-11-24T20:47:00,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220253 2024-11-24T20:47:00,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220253 2024-11-24T20:47:00,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220253) roll requested 2024-11-24T20:47:00,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220260 2024-11-24T20:47:00,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220260 2024-11-24T20:47:00,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220260) roll requested 2024-11-24T20:47:00,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220266 2024-11-24T20:47:00,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220266 2024-11-24T20:47:00,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220266) roll requested 2024-11-24T20:47:00,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220271 2024-11-24T20:47:00,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220271 2024-11-24T20:47:00,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220271) roll requested 2024-11-24T20:47:00,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220277 2024-11-24T20:47:00,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220277 2024-11-24T20:47:00,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220277) roll requested 2024-11-24T20:47:00,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220283 2024-11-24T20:47:00,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220283 2024-11-24T20:47:00,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220283) roll requested 2024-11-24T20:47:00,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220289 2024-11-24T20:47:00,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220289 2024-11-24T20:47:00,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220289) roll requested 2024-11-24T20:47:00,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220295 2024-11-24T20:47:00,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220295 2024-11-24T20:47:00,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220295) roll requested 2024-11-24T20:47:00,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220301 2024-11-24T20:47:00,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220301 2024-11-24T20:47:00,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220301) roll requested 2024-11-24T20:47:00,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220307 2024-11-24T20:47:00,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220307 2024-11-24T20:47:00,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220307) roll requested 2024-11-24T20:47:00,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220312 2024-11-24T20:47:00,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220312 2024-11-24T20:47:00,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220312) roll requested 2024-11-24T20:47:00,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220317 2024-11-24T20:47:00,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220317 2024-11-24T20:47:00,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220317) roll requested 2024-11-24T20:47:00,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220322 2024-11-24T20:47:00,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220322 2024-11-24T20:47:00,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220322) roll requested 2024-11-24T20:47:00,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220328 2024-11-24T20:47:00,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220328 2024-11-24T20:47:00,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220328) roll requested 2024-11-24T20:47:00,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220334 2024-11-24T20:47:00,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220334 2024-11-24T20:47:00,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220334) roll requested 2024-11-24T20:47:00,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220339 2024-11-24T20:47:00,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220339 2024-11-24T20:47:00,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220339) roll requested 2024-11-24T20:47:00,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220344 2024-11-24T20:47:00,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220344 2024-11-24T20:47:00,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220344) roll requested 2024-11-24T20:47:00,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220350 2024-11-24T20:47:00,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220350 2024-11-24T20:47:00,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220350) roll requested 2024-11-24T20:47:00,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220356 2024-11-24T20:47:00,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220356 2024-11-24T20:47:00,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220356) roll requested 2024-11-24T20:47:00,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220364 2024-11-24T20:47:00,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220364 2024-11-24T20:47:00,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220364) roll requested 2024-11-24T20:47:00,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220373 2024-11-24T20:47:00,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220373 2024-11-24T20:47:00,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220373) roll requested 2024-11-24T20:47:00,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220389 2024-11-24T20:47:00,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220389 2024-11-24T20:47:00,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220389) roll requested 2024-11-24T20:47:00,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220394 2024-11-24T20:47:00,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220394 2024-11-24T20:47:00,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220394) roll requested 2024-11-24T20:47:00,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220399 2024-11-24T20:47:00,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220399 2024-11-24T20:47:00,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220399) roll requested 2024-11-24T20:47:00,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220404 2024-11-24T20:47:00,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220404 2024-11-24T20:47:00,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220404) roll requested 2024-11-24T20:47:00,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220410 2024-11-24T20:47:00,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220410 2024-11-24T20:47:00,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220410) roll requested 2024-11-24T20:47:00,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220421 2024-11-24T20:47:00,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220421 2024-11-24T20:47:00,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220421) roll requested 2024-11-24T20:47:00,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220429 2024-11-24T20:47:00,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220429 2024-11-24T20:47:00,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220429) roll requested 2024-11-24T20:47:00,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220437 2024-11-24T20:47:00,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220437 2024-11-24T20:47:00,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220437) roll requested 2024-11-24T20:47:00,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220444 2024-11-24T20:47:00,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220444 2024-11-24T20:47:00,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220444) roll requested 2024-11-24T20:47:00,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220449 2024-11-24T20:47:00,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220449 2024-11-24T20:47:00,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220449) roll requested 2024-11-24T20:47:00,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220454 2024-11-24T20:47:00,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220454 2024-11-24T20:47:00,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220454) roll requested 2024-11-24T20:47:00,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220461 2024-11-24T20:47:00,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220461 2024-11-24T20:47:00,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220461) roll requested 2024-11-24T20:47:00,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220468 2024-11-24T20:47:00,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220468 2024-11-24T20:47:00,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220468) roll requested 2024-11-24T20:47:00,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220478 2024-11-24T20:47:00,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220478 2024-11-24T20:47:00,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220478) roll requested 2024-11-24T20:47:00,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220484 2024-11-24T20:47:00,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220484 2024-11-24T20:47:00,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220484) roll requested 2024-11-24T20:47:00,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220489 2024-11-24T20:47:00,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220489 2024-11-24T20:47:00,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220489) roll requested 2024-11-24T20:47:00,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220494 2024-11-24T20:47:00,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220494 2024-11-24T20:47:00,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220494) roll requested 2024-11-24T20:47:00,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220500 2024-11-24T20:47:00,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220500 2024-11-24T20:47:00,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220500) roll requested 2024-11-24T20:47:00,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220508 2024-11-24T20:47:00,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220508 2024-11-24T20:47:00,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220508) roll requested 2024-11-24T20:47:00,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220515 2024-11-24T20:47:00,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220515 2024-11-24T20:47:00,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220515) roll requested 2024-11-24T20:47:00,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220520 2024-11-24T20:47:00,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220520 2024-11-24T20:47:00,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220520) roll requested 2024-11-24T20:47:00,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220526 2024-11-24T20:47:00,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220526 2024-11-24T20:47:00,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220526) roll requested 2024-11-24T20:47:00,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220533 2024-11-24T20:47:00,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220533 2024-11-24T20:47:00,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220533) roll requested 2024-11-24T20:47:00,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220540 2024-11-24T20:47:00,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220540 2024-11-24T20:47:00,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220540) roll requested 2024-11-24T20:47:00,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220548 2024-11-24T20:47:00,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220548 2024-11-24T20:47:00,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220548) roll requested 2024-11-24T20:47:00,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220558 2024-11-24T20:47:00,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220558 2024-11-24T20:47:00,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220558) roll requested 2024-11-24T20:47:00,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220567 2024-11-24T20:47:00,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220567 2024-11-24T20:47:00,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220567) roll requested 2024-11-24T20:47:00,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220575 2024-11-24T20:47:00,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220575 2024-11-24T20:47:00,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220575) roll requested 2024-11-24T20:47:00,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220580 2024-11-24T20:47:00,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220580 2024-11-24T20:47:00,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220580) roll requested 2024-11-24T20:47:00,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220585 2024-11-24T20:47:00,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220585 2024-11-24T20:47:00,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220585) roll requested 2024-11-24T20:47:00,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220591 2024-11-24T20:47:00,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220591 2024-11-24T20:47:00,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220591) roll requested 2024-11-24T20:47:00,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220598 2024-11-24T20:47:00,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220598 2024-11-24T20:47:00,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220598) roll requested 2024-11-24T20:47:00,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220606 2024-11-24T20:47:00,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220606 2024-11-24T20:47:00,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220606) roll requested 2024-11-24T20:47:00,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220614 2024-11-24T20:47:00,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220614 2024-11-24T20:47:00,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220614) roll requested 2024-11-24T20:47:00,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220621 2024-11-24T20:47:00,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220621 2024-11-24T20:47:00,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220621) roll requested 2024-11-24T20:47:00,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220628 2024-11-24T20:47:00,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220628 2024-11-24T20:47:00,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220628) roll requested 2024-11-24T20:47:00,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220636 2024-11-24T20:47:00,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220636 2024-11-24T20:47:00,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220636) roll requested 2024-11-24T20:47:00,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220642 2024-11-24T20:47:00,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220642 2024-11-24T20:47:00,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220642) roll requested 2024-11-24T20:47:00,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220649 2024-11-24T20:47:00,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220649 2024-11-24T20:47:00,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220649) roll requested 2024-11-24T20:47:00,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220656 2024-11-24T20:47:00,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220656 2024-11-24T20:47:00,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220656) roll requested 2024-11-24T20:47:00,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220664 2024-11-24T20:47:00,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220664 2024-11-24T20:47:00,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220664) roll requested 2024-11-24T20:47:00,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220672 2024-11-24T20:47:00,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220672 2024-11-24T20:47:00,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220672) roll requested 2024-11-24T20:47:00,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220680 2024-11-24T20:47:00,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220680 2024-11-24T20:47:00,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220680) roll requested 2024-11-24T20:47:00,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220688 2024-11-24T20:47:00,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220688 2024-11-24T20:47:00,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220688) roll requested 2024-11-24T20:47:00,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220697 2024-11-24T20:47:00,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220697 2024-11-24T20:47:00,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220697) roll requested 2024-11-24T20:47:00,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220706 2024-11-24T20:47:00,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220706 2024-11-24T20:47:00,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220706) roll requested 2024-11-24T20:47:00,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220716 2024-11-24T20:47:00,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220716 2024-11-24T20:47:00,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220716) roll requested 2024-11-24T20:47:00,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220725 2024-11-24T20:47:00,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220725 2024-11-24T20:47:00,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220725) roll requested 2024-11-24T20:47:00,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220735 2024-11-24T20:47:00,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220735 2024-11-24T20:47:00,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220735) roll requested 2024-11-24T20:47:00,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220743 2024-11-24T20:47:00,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220743 2024-11-24T20:47:00,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220743) roll requested 2024-11-24T20:47:00,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220751 2024-11-24T20:47:00,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220751 2024-11-24T20:47:00,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220751) roll requested 2024-11-24T20:47:00,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220759 2024-11-24T20:47:00,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220759 2024-11-24T20:47:00,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220759) roll requested 2024-11-24T20:47:00,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220768 2024-11-24T20:47:00,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220768 2024-11-24T20:47:00,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220768) roll requested 2024-11-24T20:47:00,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220776 2024-11-24T20:47:00,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220776 2024-11-24T20:47:00,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220776) roll requested 2024-11-24T20:47:00,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220786 2024-11-24T20:47:00,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220786 2024-11-24T20:47:00,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220786) roll requested 2024-11-24T20:47:00,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220795 2024-11-24T20:47:00,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220795 2024-11-24T20:47:00,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220795) roll requested 2024-11-24T20:47:00,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220804 2024-11-24T20:47:00,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220804 2024-11-24T20:47:00,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220804) roll requested 2024-11-24T20:47:00,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220813 2024-11-24T20:47:00,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220813 2024-11-24T20:47:00,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220813) roll requested 2024-11-24T20:47:00,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220821 2024-11-24T20:47:00,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220821 2024-11-24T20:47:00,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220821) roll requested 2024-11-24T20:47:00,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220835 2024-11-24T20:47:00,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220835 2024-11-24T20:47:00,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220835) roll requested 2024-11-24T20:47:00,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220840 2024-11-24T20:47:00,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220840 2024-11-24T20:47:00,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220840) roll requested 2024-11-24T20:47:00,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220845 2024-11-24T20:47:00,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220845 2024-11-24T20:47:00,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220840 is not closed yet, will try archiving it next time 2024-11-24T20:47:00,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220845) roll requested 2024-11-24T20:47:00,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220851 2024-11-24T20:47:00,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220851 2024-11-24T20:47:00,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220851) roll requested 2024-11-24T20:47:00,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220909 2024-11-24T20:47:00,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220909 2024-11-24T20:47:00,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220909) roll requested 2024-11-24T20:47:00,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220916 2024-11-24T20:47:00,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220916 2024-11-24T20:47:00,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220916) roll requested 2024-11-24T20:47:00,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220921 2024-11-24T20:47:00,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220921 2024-11-24T20:47:00,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220921) roll requested 2024-11-24T20:47:00,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220931 2024-11-24T20:47:00,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220931 2024-11-24T20:47:00,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220931) roll requested 2024-11-24T20:47:00,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220937 2024-11-24T20:47:00,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220937 2024-11-24T20:47:00,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220937) roll requested 2024-11-24T20:47:00,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220947 2024-11-24T20:47:00,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220947 2024-11-24T20:47:00,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220947) roll requested 2024-11-24T20:47:00,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220952 2024-11-24T20:47:00,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220952 2024-11-24T20:47:00,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220952) roll requested 2024-11-24T20:47:00,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220957 2024-11-24T20:47:00,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220957 2024-11-24T20:47:00,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220957) roll requested 2024-11-24T20:47:00,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220962 2024-11-24T20:47:00,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220962 2024-11-24T20:47:00,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220962) roll requested 2024-11-24T20:47:00,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220968 2024-11-24T20:47:00,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220968 2024-11-24T20:47:00,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220968) roll requested 2024-11-24T20:47:00,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220973 2024-11-24T20:47:00,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220973 2024-11-24T20:47:00,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220973) roll requested 2024-11-24T20:47:00,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220979 2024-11-24T20:47:00,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220979 2024-11-24T20:47:00,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220979) roll requested 2024-11-24T20:47:00,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220984 2024-11-24T20:47:00,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:00,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220984 2024-11-24T20:47:00,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:00,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:00,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220984) roll requested 2024-11-24T20:47:00,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481220995 2024-11-24T20:47:01,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220995 2024-11-24T20:47:01,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481220995) roll requested 2024-11-24T20:47:01,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221001 2024-11-24T20:47:01,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481220995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221001 2024-11-24T20:47:01,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221001) roll requested 2024-11-24T20:47:01,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221006 2024-11-24T20:47:01,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221006 2024-11-24T20:47:01,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221006) roll requested 2024-11-24T20:47:01,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221011 2024-11-24T20:47:01,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221011 2024-11-24T20:47:01,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221011) roll requested 2024-11-24T20:47:01,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221017 2024-11-24T20:47:01,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221017 2024-11-24T20:47:01,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221017) roll requested 2024-11-24T20:47:01,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221022 2024-11-24T20:47:01,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221022 2024-11-24T20:47:01,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221022) roll requested 2024-11-24T20:47:01,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221027 2024-11-24T20:47:01,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221027 2024-11-24T20:47:01,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221027) roll requested 2024-11-24T20:47:01,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221036 2024-11-24T20:47:01,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221036 2024-11-24T20:47:01,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221036) roll requested 2024-11-24T20:47:01,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221043 2024-11-24T20:47:01,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221043 2024-11-24T20:47:01,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221043) roll requested 2024-11-24T20:47:01,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221048 2024-11-24T20:47:01,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221048 2024-11-24T20:47:01,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221048) roll requested 2024-11-24T20:47:01,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221055 2024-11-24T20:47:01,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221055 2024-11-24T20:47:01,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221055) roll requested 2024-11-24T20:47:01,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221062 2024-11-24T20:47:01,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221062 2024-11-24T20:47:01,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221062) roll requested 2024-11-24T20:47:01,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221070 2024-11-24T20:47:01,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221070 2024-11-24T20:47:01,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221070) roll requested 2024-11-24T20:47:01,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221079 2024-11-24T20:47:01,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221079 2024-11-24T20:47:01,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221079) roll requested 2024-11-24T20:47:01,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221087 2024-11-24T20:47:01,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221087 2024-11-24T20:47:01,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221087) roll requested 2024-11-24T20:47:01,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221097 2024-11-24T20:47:01,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221097 2024-11-24T20:47:01,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221097) roll requested 2024-11-24T20:47:01,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221113 2024-11-24T20:47:01,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221113 2024-11-24T20:47:01,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221113) roll requested 2024-11-24T20:47:01,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221122 2024-11-24T20:47:01,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221122 2024-11-24T20:47:01,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221122) roll requested 2024-11-24T20:47:01,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221131 2024-11-24T20:47:01,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221131 2024-11-24T20:47:01,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221131) roll requested 2024-11-24T20:47:01,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221138 2024-11-24T20:47:01,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221138 2024-11-24T20:47:01,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221138) roll requested 2024-11-24T20:47:01,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221143 2024-11-24T20:47:01,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221143 2024-11-24T20:47:01,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221143) roll requested 2024-11-24T20:47:01,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221149 2024-11-24T20:47:01,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221149 2024-11-24T20:47:01,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221149) roll requested 2024-11-24T20:47:01,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221155 2024-11-24T20:47:01,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221155 2024-11-24T20:47:01,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221155) roll requested 2024-11-24T20:47:01,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221162 2024-11-24T20:47:01,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221162 2024-11-24T20:47:01,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221162) roll requested 2024-11-24T20:47:01,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221171 2024-11-24T20:47:01,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221171 2024-11-24T20:47:01,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221171) roll requested 2024-11-24T20:47:01,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221182 2024-11-24T20:47:01,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221182 2024-11-24T20:47:01,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221182) roll requested 2024-11-24T20:47:01,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221192 2024-11-24T20:47:01,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221192 2024-11-24T20:47:01,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221192) roll requested 2024-11-24T20:47:01,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221201 2024-11-24T20:47:01,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221201 2024-11-24T20:47:01,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221201) roll requested 2024-11-24T20:47:01,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221206 2024-11-24T20:47:01,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221206 2024-11-24T20:47:01,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221206) roll requested 2024-11-24T20:47:01,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221211 2024-11-24T20:47:01,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221211 2024-11-24T20:47:01,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221211) roll requested 2024-11-24T20:47:01,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221219 2024-11-24T20:47:01,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221219 2024-11-24T20:47:01,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221219) roll requested 2024-11-24T20:47:01,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221227 2024-11-24T20:47:01,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221227 2024-11-24T20:47:01,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221227) roll requested 2024-11-24T20:47:01,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221238 2024-11-24T20:47:01,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221238 2024-11-24T20:47:01,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221238) roll requested 2024-11-24T20:47:01,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221246 2024-11-24T20:47:01,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221246 2024-11-24T20:47:01,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221246) roll requested 2024-11-24T20:47:01,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221264 2024-11-24T20:47:01,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221264 2024-11-24T20:47:01,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221264) roll requested 2024-11-24T20:47:01,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221272 2024-11-24T20:47:01,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221272 2024-11-24T20:47:01,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221272) roll requested 2024-11-24T20:47:01,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221281 2024-11-24T20:47:01,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221281 2024-11-24T20:47:01,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221281) roll requested 2024-11-24T20:47:01,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221287 2024-11-24T20:47:01,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221287 2024-11-24T20:47:01,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221287) roll requested 2024-11-24T20:47:01,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221293 2024-11-24T20:47:01,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221293 2024-11-24T20:47:01,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221293) roll requested 2024-11-24T20:47:01,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221304 2024-11-24T20:47:01,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221304 2024-11-24T20:47:01,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221304) roll requested 2024-11-24T20:47:01,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221312 2024-11-24T20:47:01,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221312 2024-11-24T20:47:01,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221312) roll requested 2024-11-24T20:47:01,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221319 2024-11-24T20:47:01,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221319 2024-11-24T20:47:01,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221319) roll requested 2024-11-24T20:47:01,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221325 2024-11-24T20:47:01,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221325 2024-11-24T20:47:01,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221325) roll requested 2024-11-24T20:47:01,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221330 2024-11-24T20:47:01,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221330 2024-11-24T20:47:01,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221330) roll requested 2024-11-24T20:47:01,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221336 2024-11-24T20:47:01,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221336 2024-11-24T20:47:01,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221336) roll requested 2024-11-24T20:47:01,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221341 2024-11-24T20:47:01,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221341 2024-11-24T20:47:01,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221341) roll requested 2024-11-24T20:47:01,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221347 2024-11-24T20:47:01,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221347 2024-11-24T20:47:01,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221347) roll requested 2024-11-24T20:47:01,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221357 2024-11-24T20:47:01,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221357 2024-11-24T20:47:01,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221357) roll requested 2024-11-24T20:47:01,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221363 2024-11-24T20:47:01,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221363 2024-11-24T20:47:01,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221363) roll requested 2024-11-24T20:47:01,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221372 2024-11-24T20:47:01,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221372 2024-11-24T20:47:01,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221372) roll requested 2024-11-24T20:47:01,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221379 2024-11-24T20:47:01,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221379 2024-11-24T20:47:01,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221379) roll requested 2024-11-24T20:47:01,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221385 2024-11-24T20:47:01,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221385 2024-11-24T20:47:01,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221385) roll requested 2024-11-24T20:47:01,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221400 2024-11-24T20:47:01,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221400 2024-11-24T20:47:01,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221400) roll requested 2024-11-24T20:47:01,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221413 2024-11-24T20:47:01,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221413 2024-11-24T20:47:01,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221413) roll requested 2024-11-24T20:47:01,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221429 2024-11-24T20:47:01,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221429 2024-11-24T20:47:01,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221429) roll requested 2024-11-24T20:47:01,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221437 2024-11-24T20:47:01,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221437 2024-11-24T20:47:01,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221437) roll requested 2024-11-24T20:47:01,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221442 2024-11-24T20:47:01,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221442 2024-11-24T20:47:01,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221442) roll requested 2024-11-24T20:47:01,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221448 2024-11-24T20:47:01,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221448 2024-11-24T20:47:01,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221448) roll requested 2024-11-24T20:47:01,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221453 2024-11-24T20:47:01,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221453 2024-11-24T20:47:01,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221453) roll requested 2024-11-24T20:47:01,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221460 2024-11-24T20:47:01,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221460 2024-11-24T20:47:01,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221460) roll requested 2024-11-24T20:47:01,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221465 2024-11-24T20:47:01,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221465 2024-11-24T20:47:01,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221465) roll requested 2024-11-24T20:47:01,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221471 2024-11-24T20:47:01,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221471 2024-11-24T20:47:01,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221471) roll requested 2024-11-24T20:47:01,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221477 2024-11-24T20:47:01,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221477 2024-11-24T20:47:01,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221477) roll requested 2024-11-24T20:47:01,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221484 2024-11-24T20:47:01,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221484 2024-11-24T20:47:01,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221484) roll requested 2024-11-24T20:47:01,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221490 2024-11-24T20:47:01,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221490 2024-11-24T20:47:01,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221490) roll requested 2024-11-24T20:47:01,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221494 2024-11-24T20:47:01,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221494 2024-11-24T20:47:01,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221494) roll requested 2024-11-24T20:47:01,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221499 2024-11-24T20:47:01,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221499 2024-11-24T20:47:01,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221499) roll requested 2024-11-24T20:47:01,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221504 2024-11-24T20:47:01,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221504 2024-11-24T20:47:01,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221504) roll requested 2024-11-24T20:47:01,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221511 2024-11-24T20:47:01,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221511 2024-11-24T20:47:01,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221511) roll requested 2024-11-24T20:47:01,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221518 2024-11-24T20:47:01,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221518 2024-11-24T20:47:01,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221518) roll requested 2024-11-24T20:47:01,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221525 2024-11-24T20:47:01,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221525 2024-11-24T20:47:01,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221525) roll requested 2024-11-24T20:47:01,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221530 2024-11-24T20:47:01,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221530 2024-11-24T20:47:01,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221530) roll requested 2024-11-24T20:47:01,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221536 2024-11-24T20:47:01,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221536 2024-11-24T20:47:01,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221536) roll requested 2024-11-24T20:47:01,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221547 2024-11-24T20:47:01,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221547 2024-11-24T20:47:01,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221547) roll requested 2024-11-24T20:47:01,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221553 2024-11-24T20:47:01,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221553 2024-11-24T20:47:01,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221553) roll requested 2024-11-24T20:47:01,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221560 2024-11-24T20:47:01,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221560 2024-11-24T20:47:01,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221560) roll requested 2024-11-24T20:47:01,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221566 2024-11-24T20:47:01,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221566 2024-11-24T20:47:01,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221566) roll requested 2024-11-24T20:47:01,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221573 2024-11-24T20:47:01,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221573 2024-11-24T20:47:01,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221573) roll requested 2024-11-24T20:47:01,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221582 2024-11-24T20:47:01,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221582 2024-11-24T20:47:01,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221582) roll requested 2024-11-24T20:47:01,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221590 2024-11-24T20:47:01,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221590 2024-11-24T20:47:01,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221590) roll requested 2024-11-24T20:47:01,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221597 2024-11-24T20:47:01,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221597 2024-11-24T20:47:01,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221597) roll requested 2024-11-24T20:47:01,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221605 2024-11-24T20:47:01,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221605 2024-11-24T20:47:01,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221605) roll requested 2024-11-24T20:47:01,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221614 2024-11-24T20:47:01,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221614 2024-11-24T20:47:01,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221614) roll requested 2024-11-24T20:47:01,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221624 2024-11-24T20:47:01,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221624 2024-11-24T20:47:01,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221624) roll requested 2024-11-24T20:47:01,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221635 2024-11-24T20:47:01,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221635 2024-11-24T20:47:01,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221635) roll requested 2024-11-24T20:47:01,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221645 2024-11-24T20:47:01,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221645 2024-11-24T20:47:01,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221645) roll requested 2024-11-24T20:47:01,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221654 2024-11-24T20:47:01,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221654 2024-11-24T20:47:01,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221654) roll requested 2024-11-24T20:47:01,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221662 2024-11-24T20:47:01,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221662 2024-11-24T20:47:01,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221662) roll requested 2024-11-24T20:47:01,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221671 2024-11-24T20:47:01,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221671 2024-11-24T20:47:01,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221671) roll requested 2024-11-24T20:47:01,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221680 2024-11-24T20:47:01,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221680 2024-11-24T20:47:01,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221680) roll requested 2024-11-24T20:47:01,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221690 2024-11-24T20:47:01,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221690 2024-11-24T20:47:01,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221690) roll requested 2024-11-24T20:47:01,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221698 2024-11-24T20:47:01,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221698 2024-11-24T20:47:01,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221698) roll requested 2024-11-24T20:47:01,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221704 2024-11-24T20:47:01,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221704 2024-11-24T20:47:01,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221704) roll requested 2024-11-24T20:47:01,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221712 2024-11-24T20:47:01,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221712 2024-11-24T20:47:01,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221712) roll requested 2024-11-24T20:47:01,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221721 2024-11-24T20:47:01,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221721 2024-11-24T20:47:01,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221721) roll requested 2024-11-24T20:47:01,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221730 2024-11-24T20:47:01,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221730 2024-11-24T20:47:01,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221730) roll requested 2024-11-24T20:47:01,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221740 2024-11-24T20:47:01,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221740 2024-11-24T20:47:01,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221740) roll requested 2024-11-24T20:47:01,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221750 2024-11-24T20:47:01,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221750 2024-11-24T20:47:01,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221750) roll requested 2024-11-24T20:47:01,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221758 2024-11-24T20:47:01,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221758 2024-11-24T20:47:01,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221758) roll requested 2024-11-24T20:47:01,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221766 2024-11-24T20:47:01,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221766 2024-11-24T20:47:01,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221766) roll requested 2024-11-24T20:47:01,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221775 2024-11-24T20:47:01,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221775 2024-11-24T20:47:01,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221775) roll requested 2024-11-24T20:47:01,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221788 2024-11-24T20:47:01,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221788 2024-11-24T20:47:01,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221788) roll requested 2024-11-24T20:47:01,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221796 2024-11-24T20:47:01,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221796 2024-11-24T20:47:01,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221796) roll requested 2024-11-24T20:47:01,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221820 2024-11-24T20:47:01,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221820 2024-11-24T20:47:01,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221820) roll requested 2024-11-24T20:47:01,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221828 2024-11-24T20:47:01,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221828 2024-11-24T20:47:01,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221828) roll requested 2024-11-24T20:47:01,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221837 2024-11-24T20:47:01,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221837 2024-11-24T20:47:01,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221837) roll requested 2024-11-24T20:47:01,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221913 2024-11-24T20:47:01,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:01,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221913 2024-11-24T20:47:01,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:01,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:01,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221913) roll requested 2024-11-24T20:47:01,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481221987 2024-11-24T20:47:02,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221987 2024-11-24T20:47:02,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481221987) roll requested 2024-11-24T20:47:02,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222038 2024-11-24T20:47:02,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481221987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222038 2024-11-24T20:47:02,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222038) roll requested 2024-11-24T20:47:02,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222064 2024-11-24T20:47:02,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222064 2024-11-24T20:47:02,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222064) roll requested 2024-11-24T20:47:02,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222097 2024-11-24T20:47:02,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222097 2024-11-24T20:47:02,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222097) roll requested 2024-11-24T20:47:02,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222119 2024-11-24T20:47:02,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222119 2024-11-24T20:47:02,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222119) roll requested 2024-11-24T20:47:02,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222146 2024-11-24T20:47:02,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222146 2024-11-24T20:47:02,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222146) roll requested 2024-11-24T20:47:02,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222178 2024-11-24T20:47:02,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222178 2024-11-24T20:47:02,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222178) roll requested 2024-11-24T20:47:02,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222216 2024-11-24T20:47:02,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222216 2024-11-24T20:47:02,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222216) roll requested 2024-11-24T20:47:02,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222245 2024-11-24T20:47:02,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222245 2024-11-24T20:47:02,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222245) roll requested 2024-11-24T20:47:02,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222281 2024-11-24T20:47:02,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222281 2024-11-24T20:47:02,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222281) roll requested 2024-11-24T20:47:02,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222320 2024-11-24T20:47:02,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222320 2024-11-24T20:47:02,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222320) roll requested 2024-11-24T20:47:02,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222357 2024-11-24T20:47:02,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222357 2024-11-24T20:47:02,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222357) roll requested 2024-11-24T20:47:02,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222427 2024-11-24T20:47:02,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222427 2024-11-24T20:47:02,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222427) roll requested 2024-11-24T20:47:02,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222517 2024-11-24T20:47:02,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222517 2024-11-24T20:47:02,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222517) roll requested 2024-11-24T20:47:02,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222556 2024-11-24T20:47:02,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222556 2024-11-24T20:47:02,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222556) roll requested 2024-11-24T20:47:02,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222589 2024-11-24T20:47:02,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222589 2024-11-24T20:47:02,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222589) roll requested 2024-11-24T20:47:02,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222625 2024-11-24T20:47:02,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222625 2024-11-24T20:47:02,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222625) roll requested 2024-11-24T20:47:02,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222661 2024-11-24T20:47:02,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222661 2024-11-24T20:47:02,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222661) roll requested 2024-11-24T20:47:02,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222711 2024-11-24T20:47:02,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222711 2024-11-24T20:47:02,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222711) roll requested 2024-11-24T20:47:02,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222735 2024-11-24T20:47:02,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222735 2024-11-24T20:47:02,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222735) roll requested 2024-11-24T20:47:02,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222765 2024-11-24T20:47:02,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222765 2024-11-24T20:47:02,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222765) roll requested 2024-11-24T20:47:02,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222815 2024-11-24T20:47:02,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222815 2024-11-24T20:47:02,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222815) roll requested 2024-11-24T20:47:02,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222844 2024-11-24T20:47:02,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222844 2024-11-24T20:47:02,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222844) roll requested 2024-11-24T20:47:02,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222889 2024-11-24T20:47:02,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222889 2024-11-24T20:47:02,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222889) roll requested 2024-11-24T20:47:02,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222906 2024-11-24T20:47:02,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222906 2024-11-24T20:47:02,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222906) roll requested 2024-11-24T20:47:02,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222912 2024-11-24T20:47:02,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222912 2024-11-24T20:47:02,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222912) roll requested 2024-11-24T20:47:02,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222932 2024-11-24T20:47:02,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:02,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222932 2024-11-24T20:47:02,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:02,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222932) roll requested 2024-11-24T20:47:02,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481222977 2024-11-24T20:47:03,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222977 2024-11-24T20:47:03,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481222977) roll requested 2024-11-24T20:47:03,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223057 2024-11-24T20:47:03,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481222977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223057 2024-11-24T20:47:03,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223057) roll requested 2024-11-24T20:47:03,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223096 2024-11-24T20:47:03,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223096 2024-11-24T20:47:03,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223096) roll requested 2024-11-24T20:47:03,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223121 2024-11-24T20:47:03,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223121 2024-11-24T20:47:03,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223121) roll requested 2024-11-24T20:47:03,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223153 2024-11-24T20:47:03,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223153 2024-11-24T20:47:03,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223153) roll requested 2024-11-24T20:47:03,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223169 2024-11-24T20:47:03,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223169 2024-11-24T20:47:03,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223169) roll requested 2024-11-24T20:47:03,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223217 2024-11-24T20:47:03,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223217 2024-11-24T20:47:03,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223217) roll requested 2024-11-24T20:47:03,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223255 2024-11-24T20:47:03,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223255 2024-11-24T20:47:03,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223255) roll requested 2024-11-24T20:47:03,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223292 2024-11-24T20:47:03,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223292 2024-11-24T20:47:03,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223292) roll requested 2024-11-24T20:47:03,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223312 2024-11-24T20:47:03,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223312 2024-11-24T20:47:03,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223312) roll requested 2024-11-24T20:47:03,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223324 2024-11-24T20:47:03,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223324 2024-11-24T20:47:03,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223324) roll requested 2024-11-24T20:47:03,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223344 2024-11-24T20:47:03,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223344 2024-11-24T20:47:03,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223344) roll requested 2024-11-24T20:47:03,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223357 2024-11-24T20:47:03,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223357 2024-11-24T20:47:03,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223357) roll requested 2024-11-24T20:47:03,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223373 2024-11-24T20:47:03,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223373 2024-11-24T20:47:03,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223373) roll requested 2024-11-24T20:47:03,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223384 2024-11-24T20:47:03,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223384 2024-11-24T20:47:03,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223384) roll requested 2024-11-24T20:47:03,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223392 2024-11-24T20:47:03,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223392 2024-11-24T20:47:03,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223392) roll requested 2024-11-24T20:47:03,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223413 2024-11-24T20:47:03,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223413 2024-11-24T20:47:03,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223413) roll requested 2024-11-24T20:47:03,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223433 2024-11-24T20:47:03,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223433 2024-11-24T20:47:03,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223433) roll requested 2024-11-24T20:47:03,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223458 2024-11-24T20:47:03,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223458 2024-11-24T20:47:03,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223458) roll requested 2024-11-24T20:47:03,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223477 2024-11-24T20:47:03,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223477 2024-11-24T20:47:03,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223477) roll requested 2024-11-24T20:47:03,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223485 2024-11-24T20:47:03,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223485 2024-11-24T20:47:03,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223485) roll requested 2024-11-24T20:47:03,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223500 2024-11-24T20:47:03,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223500 2024-11-24T20:47:03,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223500) roll requested 2024-11-24T20:47:03,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223512 2024-11-24T20:47:03,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223512 2024-11-24T20:47:03,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223512) roll requested 2024-11-24T20:47:03,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223571 2024-11-24T20:47:03,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223571 2024-11-24T20:47:03,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223571) roll requested 2024-11-24T20:47:03,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223584 2024-11-24T20:47:03,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223584 2024-11-24T20:47:03,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223584) roll requested 2024-11-24T20:47:03,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223610 2024-11-24T20:47:03,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223610 2024-11-24T20:47:03,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223610) roll requested 2024-11-24T20:47:03,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223641 2024-11-24T20:47:03,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223641 2024-11-24T20:47:03,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223641) roll requested 2024-11-24T20:47:03,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223670 2024-11-24T20:47:03,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223670 2024-11-24T20:47:03,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223670) roll requested 2024-11-24T20:47:03,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223695 2024-11-24T20:47:03,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223695 2024-11-24T20:47:03,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223695) roll requested 2024-11-24T20:47:03,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223738 2024-11-24T20:47:03,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223738 2024-11-24T20:47:03,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223738) roll requested 2024-11-24T20:47:03,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223774 2024-11-24T20:47:03,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223774 2024-11-24T20:47:03,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223774) roll requested 2024-11-24T20:47:03,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223797 2024-11-24T20:47:03,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223797 2024-11-24T20:47:03,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223797) roll requested 2024-11-24T20:47:03,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223828 2024-11-24T20:47:03,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223828 2024-11-24T20:47:03,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223828) roll requested 2024-11-24T20:47:03,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223845 2024-11-24T20:47:03,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223845 2024-11-24T20:47:03,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223845) roll requested 2024-11-24T20:47:03,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223852 2024-11-24T20:47:03,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223852 2024-11-24T20:47:03,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223852) roll requested 2024-11-24T20:47:03,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223876 2024-11-24T20:47:03,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223876 2024-11-24T20:47:03,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223876) roll requested 2024-11-24T20:47:03,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223896 2024-11-24T20:47:03,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223896 2024-11-24T20:47:03,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223896) roll requested 2024-11-24T20:47:03,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223905 2024-11-24T20:47:03,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223905 2024-11-24T20:47:03,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223905) roll requested 2024-11-24T20:47:03,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223932 2024-11-24T20:47:03,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:03,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223932 2024-11-24T20:47:03,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:03,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223932) roll requested 2024-11-24T20:47:03,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481223944 2024-11-24T20:47:04,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223944 2024-11-24T20:47:04,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481223944) roll requested 2024-11-24T20:47:04,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224007 2024-11-24T20:47:04,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481223944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224007 2024-11-24T20:47:04,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224007) roll requested 2024-11-24T20:47:04,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224064 2024-11-24T20:47:04,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224064 2024-11-24T20:47:04,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224064) roll requested 2024-11-24T20:47:04,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224097 2024-11-24T20:47:04,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224097 2024-11-24T20:47:04,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224097) roll requested 2024-11-24T20:47:04,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224128 2024-11-24T20:47:04,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224128 2024-11-24T20:47:04,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224128) roll requested 2024-11-24T20:47:04,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224177 2024-11-24T20:47:04,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224177 2024-11-24T20:47:04,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224177) roll requested 2024-11-24T20:47:04,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224203 2024-11-24T20:47:04,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224203 2024-11-24T20:47:04,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224203) roll requested 2024-11-24T20:47:04,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224216 2024-11-24T20:47:04,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224216 2024-11-24T20:47:04,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224216) roll requested 2024-11-24T20:47:04,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224233 2024-11-24T20:47:04,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224233 2024-11-24T20:47:04,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224233) roll requested 2024-11-24T20:47:04,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224241 2024-11-24T20:47:04,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224241 2024-11-24T20:47:04,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224241) roll requested 2024-11-24T20:47:04,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224249 2024-11-24T20:47:04,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224249 2024-11-24T20:47:04,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224249) roll requested 2024-11-24T20:47:04,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224256 2024-11-24T20:47:04,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224256 2024-11-24T20:47:04,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224256) roll requested 2024-11-24T20:47:04,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224264 2024-11-24T20:47:04,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224264 2024-11-24T20:47:04,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224264) roll requested 2024-11-24T20:47:04,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224270 2024-11-24T20:47:04,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224270 2024-11-24T20:47:04,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224270) roll requested 2024-11-24T20:47:04,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224287 2024-11-24T20:47:04,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224287 2024-11-24T20:47:04,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224287) roll requested 2024-11-24T20:47:04,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224308 2024-11-24T20:47:04,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224308 2024-11-24T20:47:04,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224308) roll requested 2024-11-24T20:47:04,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224320 2024-11-24T20:47:04,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224320 2024-11-24T20:47:04,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224320) roll requested 2024-11-24T20:47:04,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224336 2024-11-24T20:47:04,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224336 2024-11-24T20:47:04,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224336) roll requested 2024-11-24T20:47:04,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224349 2024-11-24T20:47:04,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224349 2024-11-24T20:47:04,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224349) roll requested 2024-11-24T20:47:04,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224361 2024-11-24T20:47:04,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224361 2024-11-24T20:47:04,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224361) roll requested 2024-11-24T20:47:04,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224376 2024-11-24T20:47:04,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224376 2024-11-24T20:47:04,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224376) roll requested 2024-11-24T20:47:04,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224383 2024-11-24T20:47:04,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224383 2024-11-24T20:47:04,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224383) roll requested 2024-11-24T20:47:04,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224392 2024-11-24T20:47:04,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224392 2024-11-24T20:47:04,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224392) roll requested 2024-11-24T20:47:04,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224400 2024-11-24T20:47:04,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224400 2024-11-24T20:47:04,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224400) roll requested 2024-11-24T20:47:04,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224408 2024-11-24T20:47:04,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224408 2024-11-24T20:47:04,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224408) roll requested 2024-11-24T20:47:04,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224416 2024-11-24T20:47:04,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224416 2024-11-24T20:47:04,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224416) roll requested 2024-11-24T20:47:04,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224427 2024-11-24T20:47:04,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224427 2024-11-24T20:47:04,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224427) roll requested 2024-11-24T20:47:04,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224434 2024-11-24T20:47:04,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224434 2024-11-24T20:47:04,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224434) roll requested 2024-11-24T20:47:04,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224440 2024-11-24T20:47:04,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224440 2024-11-24T20:47:04,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224440) roll requested 2024-11-24T20:47:04,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224446 2024-11-24T20:47:04,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224446 2024-11-24T20:47:04,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224446) roll requested 2024-11-24T20:47:04,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224452 2024-11-24T20:47:04,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224452 2024-11-24T20:47:04,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224452) roll requested 2024-11-24T20:47:04,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224463 2024-11-24T20:47:04,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224463 2024-11-24T20:47:04,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224463) roll requested 2024-11-24T20:47:04,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224469 2024-11-24T20:47:04,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224469 2024-11-24T20:47:04,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224469) roll requested 2024-11-24T20:47:04,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224476 2024-11-24T20:47:04,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224476 2024-11-24T20:47:04,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224476) roll requested 2024-11-24T20:47:04,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224484 2024-11-24T20:47:04,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224484 2024-11-24T20:47:04,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224484) roll requested 2024-11-24T20:47:04,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224505 2024-11-24T20:47:04,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224505 2024-11-24T20:47:04,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224505) roll requested 2024-11-24T20:47:04,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224532 2024-11-24T20:47:04,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224532 2024-11-24T20:47:04,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224532) roll requested 2024-11-24T20:47:04,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224539 2024-11-24T20:47:04,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224539 2024-11-24T20:47:04,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224539) roll requested 2024-11-24T20:47:04,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224548 2024-11-24T20:47:04,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224548 2024-11-24T20:47:04,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224548) roll requested 2024-11-24T20:47:04,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224565 2024-11-24T20:47:04,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224565 2024-11-24T20:47:04,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224565) roll requested 2024-11-24T20:47:04,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224576 2024-11-24T20:47:04,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224576 2024-11-24T20:47:04,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224576) roll requested 2024-11-24T20:47:04,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224581 2024-11-24T20:47:04,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224581 2024-11-24T20:47:04,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224581) roll requested 2024-11-24T20:47:04,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224587 2024-11-24T20:47:04,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224587 2024-11-24T20:47:04,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224587) roll requested 2024-11-24T20:47:04,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224596 2024-11-24T20:47:04,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224596 2024-11-24T20:47:04,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224596) roll requested 2024-11-24T20:47:04,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224604 2024-11-24T20:47:04,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224604 2024-11-24T20:47:04,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224604) roll requested 2024-11-24T20:47:04,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224611 2024-11-24T20:47:04,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224611 2024-11-24T20:47:04,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224611) roll requested 2024-11-24T20:47:04,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224617 2024-11-24T20:47:04,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224617 2024-11-24T20:47:04,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224617) roll requested 2024-11-24T20:47:04,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224633 2024-11-24T20:47:04,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224633 2024-11-24T20:47:04,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224633) roll requested 2024-11-24T20:47:04,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224649 2024-11-24T20:47:04,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224649 2024-11-24T20:47:04,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224649) roll requested 2024-11-24T20:47:04,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224660 2024-11-24T20:47:04,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224660 2024-11-24T20:47:04,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224660) roll requested 2024-11-24T20:47:04,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224672 2024-11-24T20:47:04,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224672 2024-11-24T20:47:04,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224672) roll requested 2024-11-24T20:47:04,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224685 2024-11-24T20:47:04,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224685 2024-11-24T20:47:04,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224685) roll requested 2024-11-24T20:47:04,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224692 2024-11-24T20:47:04,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224692 2024-11-24T20:47:04,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224692) roll requested 2024-11-24T20:47:04,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224704 2024-11-24T20:47:04,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224704 2024-11-24T20:47:04,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224704) roll requested 2024-11-24T20:47:04,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224714 2024-11-24T20:47:04,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224714 2024-11-24T20:47:04,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224714) roll requested 2024-11-24T20:47:04,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224728 2024-11-24T20:47:04,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224728 2024-11-24T20:47:04,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224728) roll requested 2024-11-24T20:47:04,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224753 2024-11-24T20:47:04,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224753 2024-11-24T20:47:04,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224753) roll requested 2024-11-24T20:47:04,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224764 2024-11-24T20:47:04,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224764 2024-11-24T20:47:04,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224764) roll requested 2024-11-24T20:47:04,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224772 2024-11-24T20:47:04,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224772 2024-11-24T20:47:04,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224772) roll requested 2024-11-24T20:47:04,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224784 2024-11-24T20:47:04,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224784 2024-11-24T20:47:04,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224784) roll requested 2024-11-24T20:47:04,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224796 2024-11-24T20:47:04,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224796 2024-11-24T20:47:04,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224796) roll requested 2024-11-24T20:47:04,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224808 2024-11-24T20:47:04,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224808 2024-11-24T20:47:04,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224808) roll requested 2024-11-24T20:47:04,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224821 2024-11-24T20:47:04,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224821 2024-11-24T20:47:04,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224821) roll requested 2024-11-24T20:47:04,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224832 2024-11-24T20:47:04,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224832 2024-11-24T20:47:04,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224832) roll requested 2024-11-24T20:47:04,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224848 2024-11-24T20:47:04,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224848 2024-11-24T20:47:04,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224848) roll requested 2024-11-24T20:47:04,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224858 2024-11-24T20:47:04,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224858 2024-11-24T20:47:04,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224858) roll requested 2024-11-24T20:47:04,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224864 2024-11-24T20:47:04,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224864 2024-11-24T20:47:04,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224864) roll requested 2024-11-24T20:47:04,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224872 2024-11-24T20:47:04,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224872 2024-11-24T20:47:04,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224872) roll requested 2024-11-24T20:47:04,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224888 2024-11-24T20:47:04,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224888 2024-11-24T20:47:04,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224888) roll requested 2024-11-24T20:47:04,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224896 2024-11-24T20:47:04,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224896 2024-11-24T20:47:04,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224896) roll requested 2024-11-24T20:47:04,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224910 2024-11-24T20:47:04,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224910 2024-11-24T20:47:04,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224910) roll requested 2024-11-24T20:47:04,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224922 2024-11-24T20:47:04,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224922 2024-11-24T20:47:04,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224922) roll requested 2024-11-24T20:47:04,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224931 2024-11-24T20:47:04,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224931 2024-11-24T20:47:04,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224931) roll requested 2024-11-24T20:47:04,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224945 2024-11-24T20:47:04,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224945 2024-11-24T20:47:04,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224945) roll requested 2024-11-24T20:47:04,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224964 2024-11-24T20:47:04,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224964 2024-11-24T20:47:04,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224964) roll requested 2024-11-24T20:47:04,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224972 2024-11-24T20:47:04,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224972 2024-11-24T20:47:04,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224972) roll requested 2024-11-24T20:47:04,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224980 2024-11-24T20:47:04,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224980 2024-11-24T20:47:04,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224980) roll requested 2024-11-24T20:47:04,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224988 2024-11-24T20:47:04,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:04,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224988 2024-11-24T20:47:04,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:04,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:04,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224988) roll requested 2024-11-24T20:47:04,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481224996 2024-11-24T20:47:05,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224996 2024-11-24T20:47:05,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481224996) roll requested 2024-11-24T20:47:05,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225002 2024-11-24T20:47:05,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481224996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225002 2024-11-24T20:47:05,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225002) roll requested 2024-11-24T20:47:05,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225013 2024-11-24T20:47:05,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225013 2024-11-24T20:47:05,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225013) roll requested 2024-11-24T20:47:05,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225020 2024-11-24T20:47:05,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225020 2024-11-24T20:47:05,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225020) roll requested 2024-11-24T20:47:05,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225026 2024-11-24T20:47:05,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225026 2024-11-24T20:47:05,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225026) roll requested 2024-11-24T20:47:05,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225037 2024-11-24T20:47:05,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225037 2024-11-24T20:47:05,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225037) roll requested 2024-11-24T20:47:05,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225044 2024-11-24T20:47:05,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225044 2024-11-24T20:47:05,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225044) roll requested 2024-11-24T20:47:05,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225052 2024-11-24T20:47:05,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225052 2024-11-24T20:47:05,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225052) roll requested 2024-11-24T20:47:05,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225059 2024-11-24T20:47:05,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225059 2024-11-24T20:47:05,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225059) roll requested 2024-11-24T20:47:05,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225068 2024-11-24T20:47:05,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225068 2024-11-24T20:47:05,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225068) roll requested 2024-11-24T20:47:05,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225075 2024-11-24T20:47:05,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225075 2024-11-24T20:47:05,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225075) roll requested 2024-11-24T20:47:05,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225094 2024-11-24T20:47:05,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225094 2024-11-24T20:47:05,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225094) roll requested 2024-11-24T20:47:05,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225101 2024-11-24T20:47:05,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225101 2024-11-24T20:47:05,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225101) roll requested 2024-11-24T20:47:05,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225108 2024-11-24T20:47:05,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225108 2024-11-24T20:47:05,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225108) roll requested 2024-11-24T20:47:05,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225115 2024-11-24T20:47:05,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225115 2024-11-24T20:47:05,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225115) roll requested 2024-11-24T20:47:05,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225132 2024-11-24T20:47:05,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225132 2024-11-24T20:47:05,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225132) roll requested 2024-11-24T20:47:05,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225139 2024-11-24T20:47:05,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225139 2024-11-24T20:47:05,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225139) roll requested 2024-11-24T20:47:05,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225150 2024-11-24T20:47:05,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225150 2024-11-24T20:47:05,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225150) roll requested 2024-11-24T20:47:05,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225157 2024-11-24T20:47:05,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225157 2024-11-24T20:47:05,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225157) roll requested 2024-11-24T20:47:05,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225173 2024-11-24T20:47:05,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225173 2024-11-24T20:47:05,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225173) roll requested 2024-11-24T20:47:05,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225188 2024-11-24T20:47:05,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225188 2024-11-24T20:47:05,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225188) roll requested 2024-11-24T20:47:05,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225208 2024-11-24T20:47:05,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225208 2024-11-24T20:47:05,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225208) roll requested 2024-11-24T20:47:05,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225220 2024-11-24T20:47:05,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225220 2024-11-24T20:47:05,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225220) roll requested 2024-11-24T20:47:05,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225233 2024-11-24T20:47:05,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225233 2024-11-24T20:47:05,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225233) roll requested 2024-11-24T20:47:05,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225253 2024-11-24T20:47:05,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225253 2024-11-24T20:47:05,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225253) roll requested 2024-11-24T20:47:05,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225265 2024-11-24T20:47:05,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225265 2024-11-24T20:47:05,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225265) roll requested 2024-11-24T20:47:05,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225273 2024-11-24T20:47:05,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225273 2024-11-24T20:47:05,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225273) roll requested 2024-11-24T20:47:05,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225289 2024-11-24T20:47:05,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225289 2024-11-24T20:47:05,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225289) roll requested 2024-11-24T20:47:05,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225300 2024-11-24T20:47:05,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225300 2024-11-24T20:47:05,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225300) roll requested 2024-11-24T20:47:05,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225308 2024-11-24T20:47:05,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225308 2024-11-24T20:47:05,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225308) roll requested 2024-11-24T20:47:05,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225317 2024-11-24T20:47:05,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225317 2024-11-24T20:47:05,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225317) roll requested 2024-11-24T20:47:05,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225329 2024-11-24T20:47:05,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225329 2024-11-24T20:47:05,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225329) roll requested 2024-11-24T20:47:05,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225388 2024-11-24T20:47:05,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225388 2024-11-24T20:47:05,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225388) roll requested 2024-11-24T20:47:05,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225412 2024-11-24T20:47:05,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225412 2024-11-24T20:47:05,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225412) roll requested 2024-11-24T20:47:05,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225432 2024-11-24T20:47:05,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225432 2024-11-24T20:47:05,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225432) roll requested 2024-11-24T20:47:05,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225473 2024-11-24T20:47:05,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225473 2024-11-24T20:47:05,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225473) roll requested 2024-11-24T20:47:05,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225508 2024-11-24T20:47:05,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225508 2024-11-24T20:47:05,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225508) roll requested 2024-11-24T20:47:05,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225530 2024-11-24T20:47:05,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225530 2024-11-24T20:47:05,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225530) roll requested 2024-11-24T20:47:05,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225548 2024-11-24T20:47:05,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225548 2024-11-24T20:47:05,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225548) roll requested 2024-11-24T20:47:05,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225564 2024-11-24T20:47:05,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225564 2024-11-24T20:47:05,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225564) roll requested 2024-11-24T20:47:05,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225586 2024-11-24T20:47:05,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225586 2024-11-24T20:47:05,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225586) roll requested 2024-11-24T20:47:05,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225604 2024-11-24T20:47:05,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225604 2024-11-24T20:47:05,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225604) roll requested 2024-11-24T20:47:05,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225621 2024-11-24T20:47:05,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225621 2024-11-24T20:47:05,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225621) roll requested 2024-11-24T20:47:05,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225635 2024-11-24T20:47:05,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225635 2024-11-24T20:47:05,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225635) roll requested 2024-11-24T20:47:05,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225645 2024-11-24T20:47:05,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225645 2024-11-24T20:47:05,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225645) roll requested 2024-11-24T20:47:05,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225662 2024-11-24T20:47:05,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225662 2024-11-24T20:47:05,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225662) roll requested 2024-11-24T20:47:05,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225673 2024-11-24T20:47:05,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225673 2024-11-24T20:47:05,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225673) roll requested 2024-11-24T20:47:05,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225689 2024-11-24T20:47:05,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225689 2024-11-24T20:47:05,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225689) roll requested 2024-11-24T20:47:05,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225697 2024-11-24T20:47:05,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225697 2024-11-24T20:47:05,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225697) roll requested 2024-11-24T20:47:05,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225712 2024-11-24T20:47:05,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225712 2024-11-24T20:47:05,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225712) roll requested 2024-11-24T20:47:05,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225745 2024-11-24T20:47:05,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225745 2024-11-24T20:47:05,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225745) roll requested 2024-11-24T20:47:05,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225764 2024-11-24T20:47:05,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225764 2024-11-24T20:47:05,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225764) roll requested 2024-11-24T20:47:05,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225778 2024-11-24T20:47:05,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225778 2024-11-24T20:47:05,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225778) roll requested 2024-11-24T20:47:05,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225794 2024-11-24T20:47:05,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225794 2024-11-24T20:47:05,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225794) roll requested 2024-11-24T20:47:05,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225813 2024-11-24T20:47:05,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225813 2024-11-24T20:47:05,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225813) roll requested 2024-11-24T20:47:05,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225822 2024-11-24T20:47:05,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225822 2024-11-24T20:47:05,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225822) roll requested 2024-11-24T20:47:05,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225832 2024-11-24T20:47:05,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225832 2024-11-24T20:47:05,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225832) roll requested 2024-11-24T20:47:05,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225840 2024-11-24T20:47:05,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225840 2024-11-24T20:47:05,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225840) roll requested 2024-11-24T20:47:05,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225849 2024-11-24T20:47:05,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225849 2024-11-24T20:47:05,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225849) roll requested 2024-11-24T20:47:05,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225906 2024-11-24T20:47:05,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225906 2024-11-24T20:47:05,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225906) roll requested 2024-11-24T20:47:05,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225942 2024-11-24T20:47:05,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:05,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225942 2024-11-24T20:47:05,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:05,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:05,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225942) roll requested 2024-11-24T20:47:05,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481225984 2024-11-24T20:47:06,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225984 2024-11-24T20:47:06,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481225984) roll requested 2024-11-24T20:47:06,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226049 2024-11-24T20:47:06,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481225984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226049 2024-11-24T20:47:06,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226049) roll requested 2024-11-24T20:47:06,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226073 2024-11-24T20:47:06,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226073 2024-11-24T20:47:06,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226073) roll requested 2024-11-24T20:47:06,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226117 2024-11-24T20:47:06,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226117 2024-11-24T20:47:06,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226117) roll requested 2024-11-24T20:47:06,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226170 2024-11-24T20:47:06,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226170 2024-11-24T20:47:06,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226170) roll requested 2024-11-24T20:47:06,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226193 2024-11-24T20:47:06,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226193 2024-11-24T20:47:06,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226193) roll requested 2024-11-24T20:47:06,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226205 2024-11-24T20:47:06,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226205 2024-11-24T20:47:06,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226205) roll requested 2024-11-24T20:47:06,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226214 2024-11-24T20:47:06,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226214 2024-11-24T20:47:06,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226214) roll requested 2024-11-24T20:47:06,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226230 2024-11-24T20:47:06,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226230 2024-11-24T20:47:06,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226230) roll requested 2024-11-24T20:47:06,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226244 2024-11-24T20:47:06,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226244 2024-11-24T20:47:06,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226244) roll requested 2024-11-24T20:47:06,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226273 2024-11-24T20:47:06,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226273 2024-11-24T20:47:06,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226273) roll requested 2024-11-24T20:47:06,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226293 2024-11-24T20:47:06,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226293 2024-11-24T20:47:06,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226293) roll requested 2024-11-24T20:47:06,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226318 2024-11-24T20:47:06,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226318 2024-11-24T20:47:06,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226318) roll requested 2024-11-24T20:47:06,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226359 2024-11-24T20:47:06,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226359 2024-11-24T20:47:06,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226359) roll requested 2024-11-24T20:47:06,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226376 2024-11-24T20:47:06,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226376 2024-11-24T20:47:06,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226376) roll requested 2024-11-24T20:47:06,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226396 2024-11-24T20:47:06,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226396 2024-11-24T20:47:06,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226396) roll requested 2024-11-24T20:47:06,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226414 2024-11-24T20:47:06,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226414 2024-11-24T20:47:06,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226414) roll requested 2024-11-24T20:47:06,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226432 2024-11-24T20:47:06,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226432 2024-11-24T20:47:06,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226432) roll requested 2024-11-24T20:47:06,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226449 2024-11-24T20:47:06,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226449 2024-11-24T20:47:06,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226449) roll requested 2024-11-24T20:47:06,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226457 2024-11-24T20:47:06,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226457 2024-11-24T20:47:06,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226457) roll requested 2024-11-24T20:47:06,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226465 2024-11-24T20:47:06,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226465 2024-11-24T20:47:06,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226465) roll requested 2024-11-24T20:47:06,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226481 2024-11-24T20:47:06,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226481 2024-11-24T20:47:06,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226481) roll requested 2024-11-24T20:47:06,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226488 2024-11-24T20:47:06,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226488 2024-11-24T20:47:06,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226488) roll requested 2024-11-24T20:47:06,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226500 2024-11-24T20:47:06,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226500 2024-11-24T20:47:06,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226500) roll requested 2024-11-24T20:47:06,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226508 2024-11-24T20:47:06,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226508 2024-11-24T20:47:06,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226508) roll requested 2024-11-24T20:47:06,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226528 2024-11-24T20:47:06,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226528 2024-11-24T20:47:06,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226528) roll requested 2024-11-24T20:47:06,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226541 2024-11-24T20:47:06,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226541 2024-11-24T20:47:06,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226541) roll requested 2024-11-24T20:47:06,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226548 2024-11-24T20:47:06,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226548 2024-11-24T20:47:06,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226548) roll requested 2024-11-24T20:47:06,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226556 2024-11-24T20:47:06,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226556 2024-11-24T20:47:06,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226556) roll requested 2024-11-24T20:47:06,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226564 2024-11-24T20:47:06,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226564 2024-11-24T20:47:06,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226564) roll requested 2024-11-24T20:47:06,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226571 2024-11-24T20:47:06,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226571 2024-11-24T20:47:06,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226571) roll requested 2024-11-24T20:47:06,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226579 2024-11-24T20:47:06,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226579 2024-11-24T20:47:06,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226579) roll requested 2024-11-24T20:47:06,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226601 2024-11-24T20:47:06,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226601 2024-11-24T20:47:06,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226601) roll requested 2024-11-24T20:47:06,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226609 2024-11-24T20:47:06,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226609 2024-11-24T20:47:06,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226609) roll requested 2024-11-24T20:47:06,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226628 2024-11-24T20:47:06,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226628 2024-11-24T20:47:06,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226628) roll requested 2024-11-24T20:47:06,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226637 2024-11-24T20:47:06,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226637 2024-11-24T20:47:06,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226637) roll requested 2024-11-24T20:47:06,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226645 2024-11-24T20:47:06,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226645 2024-11-24T20:47:06,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226645) roll requested 2024-11-24T20:47:06,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226676 2024-11-24T20:47:06,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226676 2024-11-24T20:47:06,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226676) roll requested 2024-11-24T20:47:06,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226689 2024-11-24T20:47:06,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226689 2024-11-24T20:47:06,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226689) roll requested 2024-11-24T20:47:06,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226697 2024-11-24T20:47:06,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226697 2024-11-24T20:47:06,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226697) roll requested 2024-11-24T20:47:06,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226711 2024-11-24T20:47:06,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226711 2024-11-24T20:47:06,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226711) roll requested 2024-11-24T20:47:06,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226724 2024-11-24T20:47:06,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226724 2024-11-24T20:47:06,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226724) roll requested 2024-11-24T20:47:06,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226745 2024-11-24T20:47:06,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226745 2024-11-24T20:47:06,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226745) roll requested 2024-11-24T20:47:06,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226756 2024-11-24T20:47:06,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226756 2024-11-24T20:47:06,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226756) roll requested 2024-11-24T20:47:06,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226764 2024-11-24T20:47:06,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226764 2024-11-24T20:47:06,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226764) roll requested 2024-11-24T20:47:06,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226780 2024-11-24T20:47:06,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226780 2024-11-24T20:47:06,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226780) roll requested 2024-11-24T20:47:06,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226805 2024-11-24T20:47:06,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226805 2024-11-24T20:47:06,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226805) roll requested 2024-11-24T20:47:06,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226825 2024-11-24T20:47:06,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226825 2024-11-24T20:47:06,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226825) roll requested 2024-11-24T20:47:06,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226844 2024-11-24T20:47:06,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226844 2024-11-24T20:47:06,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226844) roll requested 2024-11-24T20:47:06,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226856 2024-11-24T20:47:06,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226856 2024-11-24T20:47:06,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226856) roll requested 2024-11-24T20:47:06,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226872 2024-11-24T20:47:06,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226872 2024-11-24T20:47:06,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226872) roll requested 2024-11-24T20:47:06,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226888 2024-11-24T20:47:06,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226888 2024-11-24T20:47:06,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226888) roll requested 2024-11-24T20:47:06,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226907 2024-11-24T20:47:06,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226907 2024-11-24T20:47:06,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226907) roll requested 2024-11-24T20:47:06,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226914 2024-11-24T20:47:06,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226914 2024-11-24T20:47:06,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226914) roll requested 2024-11-24T20:47:06,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226924 2024-11-24T20:47:06,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226924 2024-11-24T20:47:06,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226924) roll requested 2024-11-24T20:47:06,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226932 2024-11-24T20:47:06,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226932 2024-11-24T20:47:06,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226932) roll requested 2024-11-24T20:47:06,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226943 2024-11-24T20:47:06,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226943 2024-11-24T20:47:06,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226943) roll requested 2024-11-24T20:47:06,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226956 2024-11-24T20:47:06,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226956 2024-11-24T20:47:06,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226956) roll requested 2024-11-24T20:47:06,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226964 2024-11-24T20:47:06,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226964 2024-11-24T20:47:06,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226964) roll requested 2024-11-24T20:47:06,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226974 2024-11-24T20:47:06,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226974 2024-11-24T20:47:06,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:06,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226974) roll requested 2024-11-24T20:47:06,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481226982 2024-11-24T20:47:06,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:06,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226982 2024-11-24T20:47:07,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481226982) roll requested 2024-11-24T20:47:07,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227000 2024-11-24T20:47:07,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481226982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227000 2024-11-24T20:47:07,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227000) roll requested 2024-11-24T20:47:07,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227011 2024-11-24T20:47:07,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227011 2024-11-24T20:47:07,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227011) roll requested 2024-11-24T20:47:07,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227032 2024-11-24T20:47:07,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227032 2024-11-24T20:47:07,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227032) roll requested 2024-11-24T20:47:07,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227051 2024-11-24T20:47:07,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227051 2024-11-24T20:47:07,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227051) roll requested 2024-11-24T20:47:07,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227061 2024-11-24T20:47:07,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227061 2024-11-24T20:47:07,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227061) roll requested 2024-11-24T20:47:07,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227074 2024-11-24T20:47:07,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227074 2024-11-24T20:47:07,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227074) roll requested 2024-11-24T20:47:07,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227100 2024-11-24T20:47:07,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227100 2024-11-24T20:47:07,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227100) roll requested 2024-11-24T20:47:07,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227123 2024-11-24T20:47:07,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227123 2024-11-24T20:47:07,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227123) roll requested 2024-11-24T20:47:07,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227131 2024-11-24T20:47:07,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227131 2024-11-24T20:47:07,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227131) roll requested 2024-11-24T20:47:07,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227148 2024-11-24T20:47:07,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227148 2024-11-24T20:47:07,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227148) roll requested 2024-11-24T20:47:07,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227156 2024-11-24T20:47:07,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227156 2024-11-24T20:47:07,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227156) roll requested 2024-11-24T20:47:07,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227173 2024-11-24T20:47:07,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227173 2024-11-24T20:47:07,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227173) roll requested 2024-11-24T20:47:07,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227181 2024-11-24T20:47:07,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227181 2024-11-24T20:47:07,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227181) roll requested 2024-11-24T20:47:07,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227208 2024-11-24T20:47:07,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227208 2024-11-24T20:47:07,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227208) roll requested 2024-11-24T20:47:07,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227228 2024-11-24T20:47:07,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227228 2024-11-24T20:47:07,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227228) roll requested 2024-11-24T20:47:07,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227245 2024-11-24T20:47:07,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227245 2024-11-24T20:47:07,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227245) roll requested 2024-11-24T20:47:07,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227257 2024-11-24T20:47:07,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227257 2024-11-24T20:47:07,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227257) roll requested 2024-11-24T20:47:07,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227269 2024-11-24T20:47:07,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227269 2024-11-24T20:47:07,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227269) roll requested 2024-11-24T20:47:07,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227284 2024-11-24T20:47:07,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227284 2024-11-24T20:47:07,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227284) roll requested 2024-11-24T20:47:07,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227293 2024-11-24T20:47:07,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227293 2024-11-24T20:47:07,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227293) roll requested 2024-11-24T20:47:07,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227300 2024-11-24T20:47:07,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227300 2024-11-24T20:47:07,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227300) roll requested 2024-11-24T20:47:07,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227309 2024-11-24T20:47:07,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227309 2024-11-24T20:47:07,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227309) roll requested 2024-11-24T20:47:07,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227315 2024-11-24T20:47:07,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227315 2024-11-24T20:47:07,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227315) roll requested 2024-11-24T20:47:07,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227324 2024-11-24T20:47:07,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227324 2024-11-24T20:47:07,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227324) roll requested 2024-11-24T20:47:07,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227332 2024-11-24T20:47:07,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227332 2024-11-24T20:47:07,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227332) roll requested 2024-11-24T20:47:07,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227339 2024-11-24T20:47:07,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227339 2024-11-24T20:47:07,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227339) roll requested 2024-11-24T20:47:07,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227345 2024-11-24T20:47:07,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227345 2024-11-24T20:47:07,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227345) roll requested 2024-11-24T20:47:07,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227354 2024-11-24T20:47:07,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227354 2024-11-24T20:47:07,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227354) roll requested 2024-11-24T20:47:07,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227369 2024-11-24T20:47:07,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227369 2024-11-24T20:47:07,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227369) roll requested 2024-11-24T20:47:07,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227382 2024-11-24T20:47:07,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227382 2024-11-24T20:47:07,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227382) roll requested 2024-11-24T20:47:07,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227389 2024-11-24T20:47:07,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227389 2024-11-24T20:47:07,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227389) roll requested 2024-11-24T20:47:07,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227395 2024-11-24T20:47:07,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227395 2024-11-24T20:47:07,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227395) roll requested 2024-11-24T20:47:07,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227402 2024-11-24T20:47:07,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227402 2024-11-24T20:47:07,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227402) roll requested 2024-11-24T20:47:07,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227409 2024-11-24T20:47:07,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227409 2024-11-24T20:47:07,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227409) roll requested 2024-11-24T20:47:07,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227415 2024-11-24T20:47:07,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227415 2024-11-24T20:47:07,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227415) roll requested 2024-11-24T20:47:07,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227440 2024-11-24T20:47:07,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227440 2024-11-24T20:47:07,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227440) roll requested 2024-11-24T20:47:07,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227453 2024-11-24T20:47:07,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227453 2024-11-24T20:47:07,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227453) roll requested 2024-11-24T20:47:07,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227465 2024-11-24T20:47:07,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227465 2024-11-24T20:47:07,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227465) roll requested 2024-11-24T20:47:07,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227472 2024-11-24T20:47:07,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227472 2024-11-24T20:47:07,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227472) roll requested 2024-11-24T20:47:07,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227481 2024-11-24T20:47:07,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227481 2024-11-24T20:47:07,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227481) roll requested 2024-11-24T20:47:07,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227488 2024-11-24T20:47:07,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227488 2024-11-24T20:47:07,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227488) roll requested 2024-11-24T20:47:07,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227502 2024-11-24T20:47:07,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227502 2024-11-24T20:47:07,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227502) roll requested 2024-11-24T20:47:07,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227530 2024-11-24T20:47:07,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227530 2024-11-24T20:47:07,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227530) roll requested 2024-11-24T20:47:07,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227548 2024-11-24T20:47:07,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227548 2024-11-24T20:47:07,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227548) roll requested 2024-11-24T20:47:07,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227564 2024-11-24T20:47:07,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227564 2024-11-24T20:47:07,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227564) roll requested 2024-11-24T20:47:07,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227572 2024-11-24T20:47:07,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227572 2024-11-24T20:47:07,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227572) roll requested 2024-11-24T20:47:07,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227580 2024-11-24T20:47:07,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227580 2024-11-24T20:47:07,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227580) roll requested 2024-11-24T20:47:07,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227587 2024-11-24T20:47:07,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227587 2024-11-24T20:47:07,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227587) roll requested 2024-11-24T20:47:07,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227592 2024-11-24T20:47:07,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227592 2024-11-24T20:47:07,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227592) roll requested 2024-11-24T20:47:07,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227599 2024-11-24T20:47:07,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227599 2024-11-24T20:47:07,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227599) roll requested 2024-11-24T20:47:07,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227615 2024-11-24T20:47:07,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227615 2024-11-24T20:47:07,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227615) roll requested 2024-11-24T20:47:07,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227627 2024-11-24T20:47:07,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227627 2024-11-24T20:47:07,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227627) roll requested 2024-11-24T20:47:07,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227673 2024-11-24T20:47:07,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227673 2024-11-24T20:47:07,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227673) roll requested 2024-11-24T20:47:07,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227703 2024-11-24T20:47:07,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227703 2024-11-24T20:47:07,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227703) roll requested 2024-11-24T20:47:07,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227712 2024-11-24T20:47:07,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227712 2024-11-24T20:47:07,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227712) roll requested 2024-11-24T20:47:07,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227730 2024-11-24T20:47:07,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227730 2024-11-24T20:47:07,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227730) roll requested 2024-11-24T20:47:07,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227748 2024-11-24T20:47:07,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227748 2024-11-24T20:47:07,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227748) roll requested 2024-11-24T20:47:07,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227756 2024-11-24T20:47:07,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227756 2024-11-24T20:47:07,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227756) roll requested 2024-11-24T20:47:07,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227765 2024-11-24T20:47:07,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227765 2024-11-24T20:47:07,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227765) roll requested 2024-11-24T20:47:07,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227772 2024-11-24T20:47:07,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227772 2024-11-24T20:47:07,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227772) roll requested 2024-11-24T20:47:07,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227779 2024-11-24T20:47:07,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227779 2024-11-24T20:47:07,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227779) roll requested 2024-11-24T20:47:07,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227785 2024-11-24T20:47:07,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227785 2024-11-24T20:47:07,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227785) roll requested 2024-11-24T20:47:07,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227792 2024-11-24T20:47:07,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227792 2024-11-24T20:47:07,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227792) roll requested 2024-11-24T20:47:07,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227800 2024-11-24T20:47:07,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227800 2024-11-24T20:47:07,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227800) roll requested 2024-11-24T20:47:07,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227812 2024-11-24T20:47:07,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227812 2024-11-24T20:47:07,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227812) roll requested 2024-11-24T20:47:07,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227826 2024-11-24T20:47:07,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227826 2024-11-24T20:47:07,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227826) roll requested 2024-11-24T20:47:07,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227842 2024-11-24T20:47:07,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227842 2024-11-24T20:47:07,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227842) roll requested 2024-11-24T20:47:07,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227853 2024-11-24T20:47:07,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227853 2024-11-24T20:47:07,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227853) roll requested 2024-11-24T20:47:07,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227859 2024-11-24T20:47:07,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227859 2024-11-24T20:47:07,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227859) roll requested 2024-11-24T20:47:07,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227865 2024-11-24T20:47:07,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227865 2024-11-24T20:47:07,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227865) roll requested 2024-11-24T20:47:07,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227872 2024-11-24T20:47:07,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227872 2024-11-24T20:47:07,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227872) roll requested 2024-11-24T20:47:07,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227888 2024-11-24T20:47:07,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227888 2024-11-24T20:47:07,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227888) roll requested 2024-11-24T20:47:07,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227897 2024-11-24T20:47:07,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227897 2024-11-24T20:47:07,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227897) roll requested 2024-11-24T20:47:07,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227904 2024-11-24T20:47:07,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227904 2024-11-24T20:47:07,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227904) roll requested 2024-11-24T20:47:07,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227910 2024-11-24T20:47:07,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227910 2024-11-24T20:47:07,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227910) roll requested 2024-11-24T20:47:07,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227917 2024-11-24T20:47:07,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227917 2024-11-24T20:47:07,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227917) roll requested 2024-11-24T20:47:07,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227925 2024-11-24T20:47:07,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227925 2024-11-24T20:47:07,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227925) roll requested 2024-11-24T20:47:07,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227934 2024-11-24T20:47:07,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227934 2024-11-24T20:47:07,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227934) roll requested 2024-11-24T20:47:07,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227957 2024-11-24T20:47:07,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227957 2024-11-24T20:47:07,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227957) roll requested 2024-11-24T20:47:07,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227965 2024-11-24T20:47:07,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227965 2024-11-24T20:47:07,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227965) roll requested 2024-11-24T20:47:07,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227987 2024-11-24T20:47:07,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227987 2024-11-24T20:47:07,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:07,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227987) roll requested 2024-11-24T20:47:07,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481227993 2024-11-24T20:47:07,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:07,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227993 2024-11-24T20:47:07,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:07,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481227993) roll requested 2024-11-24T20:47:08,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228000 2024-11-24T20:47:08,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481227993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228000 2024-11-24T20:47:08,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228000) roll requested 2024-11-24T20:47:08,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228005 2024-11-24T20:47:08,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228005 2024-11-24T20:47:08,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228005) roll requested 2024-11-24T20:47:08,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228011 2024-11-24T20:47:08,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228011 2024-11-24T20:47:08,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228011) roll requested 2024-11-24T20:47:08,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228020 2024-11-24T20:47:08,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228020 2024-11-24T20:47:08,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228020) roll requested 2024-11-24T20:47:08,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228026 2024-11-24T20:47:08,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228026 2024-11-24T20:47:08,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228026) roll requested 2024-11-24T20:47:08,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228033 2024-11-24T20:47:08,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228033 2024-11-24T20:47:08,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228033) roll requested 2024-11-24T20:47:08,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228038 2024-11-24T20:47:08,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228038 2024-11-24T20:47:08,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228038) roll requested 2024-11-24T20:47:08,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228048 2024-11-24T20:47:08,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228048 2024-11-24T20:47:08,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228048) roll requested 2024-11-24T20:47:08,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228056 2024-11-24T20:47:08,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228056 2024-11-24T20:47:08,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228056) roll requested 2024-11-24T20:47:08,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228063 2024-11-24T20:47:08,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228063 2024-11-24T20:47:08,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228063) roll requested 2024-11-24T20:47:08,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228071 2024-11-24T20:47:08,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228071 2024-11-24T20:47:08,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228071) roll requested 2024-11-24T20:47:08,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228088 2024-11-24T20:47:08,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228088 2024-11-24T20:47:08,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228088) roll requested 2024-11-24T20:47:08,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228108 2024-11-24T20:47:08,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228108 2024-11-24T20:47:08,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228108) roll requested 2024-11-24T20:47:08,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228160 2024-11-24T20:47:08,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228160 2024-11-24T20:47:08,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228160) roll requested 2024-11-24T20:47:08,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228172 2024-11-24T20:47:08,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228172 2024-11-24T20:47:08,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228160 is not closed yet, will try archiving it next time 2024-11-24T20:47:08,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228172) roll requested 2024-11-24T20:47:08,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228193 2024-11-24T20:47:08,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228193 2024-11-24T20:47:08,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228193) roll requested 2024-11-24T20:47:08,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228213 2024-11-24T20:47:08,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228213 2024-11-24T20:47:08,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228213) roll requested 2024-11-24T20:47:08,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228228 2024-11-24T20:47:08,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228228 2024-11-24T20:47:08,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228228) roll requested 2024-11-24T20:47:08,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228243 2024-11-24T20:47:08,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228243 2024-11-24T20:47:08,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228243) roll requested 2024-11-24T20:47:08,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228249 2024-11-24T20:47:08,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228249 2024-11-24T20:47:08,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228249) roll requested 2024-11-24T20:47:08,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228255 2024-11-24T20:47:08,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228255 2024-11-24T20:47:08,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228255) roll requested 2024-11-24T20:47:08,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228262 2024-11-24T20:47:08,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228262 2024-11-24T20:47:08,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228262) roll requested 2024-11-24T20:47:08,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228275 2024-11-24T20:47:08,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228275 2024-11-24T20:47:08,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228275) roll requested 2024-11-24T20:47:08,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228285 2024-11-24T20:47:08,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228285 2024-11-24T20:47:08,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228285) roll requested 2024-11-24T20:47:08,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228292 2024-11-24T20:47:08,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228292 2024-11-24T20:47:08,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228292) roll requested 2024-11-24T20:47:08,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228300 2024-11-24T20:47:08,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228300 2024-11-24T20:47:08,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228300) roll requested 2024-11-24T20:47:08,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228309 2024-11-24T20:47:08,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228309 2024-11-24T20:47:08,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228309) roll requested 2024-11-24T20:47:08,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228314 2024-11-24T20:47:08,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228314 2024-11-24T20:47:08,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228314) roll requested 2024-11-24T20:47:08,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228323 2024-11-24T20:47:08,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228323 2024-11-24T20:47:08,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228323) roll requested 2024-11-24T20:47:08,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228333 2024-11-24T20:47:08,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228333 2024-11-24T20:47:08,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228333) roll requested 2024-11-24T20:47:08,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228345 2024-11-24T20:47:08,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228345 2024-11-24T20:47:08,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228345) roll requested 2024-11-24T20:47:08,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228353 2024-11-24T20:47:08,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228353 2024-11-24T20:47:08,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228353) roll requested 2024-11-24T20:47:08,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228360 2024-11-24T20:47:08,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228360 2024-11-24T20:47:08,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228360) roll requested 2024-11-24T20:47:08,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228370 2024-11-24T20:47:08,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228370 2024-11-24T20:47:08,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228370) roll requested 2024-11-24T20:47:08,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228378 2024-11-24T20:47:08,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228378 2024-11-24T20:47:08,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228378) roll requested 2024-11-24T20:47:08,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228396 2024-11-24T20:47:08,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228396 2024-11-24T20:47:08,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228396) roll requested 2024-11-24T20:47:08,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228414 2024-11-24T20:47:08,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228414 2024-11-24T20:47:08,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228414) roll requested 2024-11-24T20:47:08,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228436 2024-11-24T20:47:08,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228436 2024-11-24T20:47:08,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228436) roll requested 2024-11-24T20:47:08,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228448 2024-11-24T20:47:08,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228448 2024-11-24T20:47:08,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228448) roll requested 2024-11-24T20:47:08,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228460 2024-11-24T20:47:08,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228460 2024-11-24T20:47:08,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228460) roll requested 2024-11-24T20:47:08,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228468 2024-11-24T20:47:08,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228468 2024-11-24T20:47:08,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228468) roll requested 2024-11-24T20:47:08,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228480 2024-11-24T20:47:08,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228480 2024-11-24T20:47:08,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228480) roll requested 2024-11-24T20:47:08,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228486 2024-11-24T20:47:08,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228486 2024-11-24T20:47:08,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228486) roll requested 2024-11-24T20:47:08,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228493 2024-11-24T20:47:08,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228493 2024-11-24T20:47:08,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228493) roll requested 2024-11-24T20:47:08,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228499 2024-11-24T20:47:08,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228499 2024-11-24T20:47:08,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228499) roll requested 2024-11-24T20:47:08,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228505 2024-11-24T20:47:08,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228505 2024-11-24T20:47:08,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228505) roll requested 2024-11-24T20:47:08,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228512 2024-11-24T20:47:08,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228512 2024-11-24T20:47:08,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228512) roll requested 2024-11-24T20:47:08,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228521 2024-11-24T20:47:08,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228521 2024-11-24T20:47:08,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228521) roll requested 2024-11-24T20:47:08,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228526 2024-11-24T20:47:08,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228526 2024-11-24T20:47:08,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228526) roll requested 2024-11-24T20:47:08,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228532 2024-11-24T20:47:08,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228532 2024-11-24T20:47:08,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228532) roll requested 2024-11-24T20:47:08,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228541 2024-11-24T20:47:08,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228541 2024-11-24T20:47:08,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228541) roll requested 2024-11-24T20:47:08,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228548 2024-11-24T20:47:08,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228548 2024-11-24T20:47:08,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228548) roll requested 2024-11-24T20:47:08,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228556 2024-11-24T20:47:08,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228556 2024-11-24T20:47:08,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228556) roll requested 2024-11-24T20:47:08,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228569 2024-11-24T20:47:08,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228569 2024-11-24T20:47:08,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228569) roll requested 2024-11-24T20:47:08,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228585 2024-11-24T20:47:08,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228585 2024-11-24T20:47:08,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228585) roll requested 2024-11-24T20:47:08,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228605 2024-11-24T20:47:08,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228605 2024-11-24T20:47:08,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228605) roll requested 2024-11-24T20:47:08,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228625 2024-11-24T20:47:08,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228625 2024-11-24T20:47:08,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228625) roll requested 2024-11-24T20:47:08,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228632 2024-11-24T20:47:08,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228632 2024-11-24T20:47:08,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228632) roll requested 2024-11-24T20:47:08,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228642 2024-11-24T20:47:08,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228642 2024-11-24T20:47:08,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228642) roll requested 2024-11-24T20:47:08,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228658 2024-11-24T20:47:08,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228658 2024-11-24T20:47:08,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228658) roll requested 2024-11-24T20:47:08,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228665 2024-11-24T20:47:08,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228665 2024-11-24T20:47:08,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228665) roll requested 2024-11-24T20:47:08,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228677 2024-11-24T20:47:08,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228677 2024-11-24T20:47:08,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228677) roll requested 2024-11-24T20:47:08,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228685 2024-11-24T20:47:08,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228685 2024-11-24T20:47:08,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228685) roll requested 2024-11-24T20:47:08,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228692 2024-11-24T20:47:08,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228692 2024-11-24T20:47:08,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228692) roll requested 2024-11-24T20:47:08,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228700 2024-11-24T20:47:08,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228700 2024-11-24T20:47:08,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228700) roll requested 2024-11-24T20:47:08,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228708 2024-11-24T20:47:08,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228708 2024-11-24T20:47:08,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228708) roll requested 2024-11-24T20:47:08,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228716 2024-11-24T20:47:08,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228716 2024-11-24T20:47:08,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228716) roll requested 2024-11-24T20:47:08,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228725 2024-11-24T20:47:08,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228725 2024-11-24T20:47:08,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228725) roll requested 2024-11-24T20:47:08,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228733 2024-11-24T20:47:08,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228733 2024-11-24T20:47:08,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228733) roll requested 2024-11-24T20:47:08,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228760 2024-11-24T20:47:08,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228760 2024-11-24T20:47:08,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228760) roll requested 2024-11-24T20:47:08,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228770 2024-11-24T20:47:08,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228770 2024-11-24T20:47:08,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228770) roll requested 2024-11-24T20:47:08,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228781 2024-11-24T20:47:08,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228781 2024-11-24T20:47:08,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228781) roll requested 2024-11-24T20:47:08,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228788 2024-11-24T20:47:08,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228788 2024-11-24T20:47:08,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228788) roll requested 2024-11-24T20:47:08,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228795 2024-11-24T20:47:08,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228795 2024-11-24T20:47:08,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228795) roll requested 2024-11-24T20:47:08,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228804 2024-11-24T20:47:08,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228804 2024-11-24T20:47:08,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228804) roll requested 2024-11-24T20:47:08,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228811 2024-11-24T20:47:08,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228811 2024-11-24T20:47:08,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228811) roll requested 2024-11-24T20:47:08,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228837 2024-11-24T20:47:08,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228837 2024-11-24T20:47:08,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228837) roll requested 2024-11-24T20:47:08,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228852 2024-11-24T20:47:08,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228852 2024-11-24T20:47:08,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228852) roll requested 2024-11-24T20:47:08,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228864 2024-11-24T20:47:08,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228864 2024-11-24T20:47:08,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228864) roll requested 2024-11-24T20:47:08,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228881 2024-11-24T20:47:08,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228881 2024-11-24T20:47:08,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228881) roll requested 2024-11-24T20:47:08,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228888 2024-11-24T20:47:08,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228888 2024-11-24T20:47:08,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228888) roll requested 2024-11-24T20:47:08,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228901 2024-11-24T20:47:08,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228901 2024-11-24T20:47:08,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228901) roll requested 2024-11-24T20:47:08,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228908 2024-11-24T20:47:08,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228908 2024-11-24T20:47:08,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228908) roll requested 2024-11-24T20:47:08,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228916 2024-11-24T20:47:08,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228916 2024-11-24T20:47:08,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228916) roll requested 2024-11-24T20:47:08,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228924 2024-11-24T20:47:08,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228924 2024-11-24T20:47:08,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228924) roll requested 2024-11-24T20:47:08,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228944 2024-11-24T20:47:08,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228944 2024-11-24T20:47:08,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228944) roll requested 2024-11-24T20:47:08,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228961 2024-11-24T20:47:08,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228961 2024-11-24T20:47:08,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228961) roll requested 2024-11-24T20:47:08,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228971 2024-11-24T20:47:08,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228971 2024-11-24T20:47:08,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228971) roll requested 2024-11-24T20:47:08,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228982 2024-11-24T20:47:08,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228982 2024-11-24T20:47:08,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228982) roll requested 2024-11-24T20:47:08,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228989 2024-11-24T20:47:08,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:08,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228989 2024-11-24T20:47:08,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:08,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:08,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228989) roll requested 2024-11-24T20:47:09,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481228999 2024-11-24T20:47:09,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228999 2024-11-24T20:47:09,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481228999) roll requested 2024-11-24T20:47:09,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229006 2024-11-24T20:47:09,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481228999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229006 2024-11-24T20:47:09,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229006) roll requested 2024-11-24T20:47:09,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229018 2024-11-24T20:47:09,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229018 2024-11-24T20:47:09,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229018) roll requested 2024-11-24T20:47:09,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229026 2024-11-24T20:47:09,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229026 2024-11-24T20:47:09,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229026) roll requested 2024-11-24T20:47:09,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229034 2024-11-24T20:47:09,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229034 2024-11-24T20:47:09,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229034) roll requested 2024-11-24T20:47:09,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229040 2024-11-24T20:47:09,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229040 2024-11-24T20:47:09,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229040) roll requested 2024-11-24T20:47:09,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229046 2024-11-24T20:47:09,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229046 2024-11-24T20:47:09,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229046) roll requested 2024-11-24T20:47:09,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229052 2024-11-24T20:47:09,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229052 2024-11-24T20:47:09,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229052) roll requested 2024-11-24T20:47:09,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229058 2024-11-24T20:47:09,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229058 2024-11-24T20:47:09,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229058) roll requested 2024-11-24T20:47:09,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229064 2024-11-24T20:47:09,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229064 2024-11-24T20:47:09,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229064) roll requested 2024-11-24T20:47:09,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229071 2024-11-24T20:47:09,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229071 2024-11-24T20:47:09,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229071) roll requested 2024-11-24T20:47:09,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229077 2024-11-24T20:47:09,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229077 2024-11-24T20:47:09,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229077) roll requested 2024-11-24T20:47:09,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229085 2024-11-24T20:47:09,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229085 2024-11-24T20:47:09,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229085) roll requested 2024-11-24T20:47:09,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229096 2024-11-24T20:47:09,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229096 2024-11-24T20:47:09,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229096) roll requested 2024-11-24T20:47:09,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229110 2024-11-24T20:47:09,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229110 2024-11-24T20:47:09,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229110) roll requested 2024-11-24T20:47:09,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229120 2024-11-24T20:47:09,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229120 2024-11-24T20:47:09,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229120) roll requested 2024-11-24T20:47:09,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229127 2024-11-24T20:47:09,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229127 2024-11-24T20:47:09,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229127) roll requested 2024-11-24T20:47:09,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229135 2024-11-24T20:47:09,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229135 2024-11-24T20:47:09,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229135) roll requested 2024-11-24T20:47:09,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229146 2024-11-24T20:47:09,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229146 2024-11-24T20:47:09,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229146) roll requested 2024-11-24T20:47:09,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229152 2024-11-24T20:47:09,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229152 2024-11-24T20:47:09,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229152) roll requested 2024-11-24T20:47:09,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229158 2024-11-24T20:47:09,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229158 2024-11-24T20:47:09,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229158) roll requested 2024-11-24T20:47:09,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229164 2024-11-24T20:47:09,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229164 2024-11-24T20:47:09,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229164) roll requested 2024-11-24T20:47:09,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229171 2024-11-24T20:47:09,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229171 2024-11-24T20:47:09,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229171) roll requested 2024-11-24T20:47:09,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229205 2024-11-24T20:47:09,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229205 2024-11-24T20:47:09,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229205) roll requested 2024-11-24T20:47:09,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229220 2024-11-24T20:47:09,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229220 2024-11-24T20:47:09,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229220) roll requested 2024-11-24T20:47:09,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229237 2024-11-24T20:47:09,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229237 2024-11-24T20:47:09,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229237) roll requested 2024-11-24T20:47:09,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229256 2024-11-24T20:47:09,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229256 2024-11-24T20:47:09,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229256) roll requested 2024-11-24T20:47:09,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229277 2024-11-24T20:47:09,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229277 2024-11-24T20:47:09,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229277) roll requested 2024-11-24T20:47:09,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229294 2024-11-24T20:47:09,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229294 2024-11-24T20:47:09,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229294) roll requested 2024-11-24T20:47:09,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229301 2024-11-24T20:47:09,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229301 2024-11-24T20:47:09,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229301) roll requested 2024-11-24T20:47:09,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229307 2024-11-24T20:47:09,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229307 2024-11-24T20:47:09,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229307) roll requested 2024-11-24T20:47:09,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229316 2024-11-24T20:47:09,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229316 2024-11-24T20:47:09,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229316) roll requested 2024-11-24T20:47:09,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229323 2024-11-24T20:47:09,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229323 2024-11-24T20:47:09,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229323) roll requested 2024-11-24T20:47:09,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229330 2024-11-24T20:47:09,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229330 2024-11-24T20:47:09,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229330) roll requested 2024-11-24T20:47:09,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229336 2024-11-24T20:47:09,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229336 2024-11-24T20:47:09,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229336) roll requested 2024-11-24T20:47:09,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229344 2024-11-24T20:47:09,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229344 2024-11-24T20:47:09,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229344) roll requested 2024-11-24T20:47:09,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229351 2024-11-24T20:47:09,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229351 2024-11-24T20:47:09,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229351) roll requested 2024-11-24T20:47:09,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229360 2024-11-24T20:47:09,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229360 2024-11-24T20:47:09,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229360) roll requested 2024-11-24T20:47:09,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229370 2024-11-24T20:47:09,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229370 2024-11-24T20:47:09,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229370) roll requested 2024-11-24T20:47:09,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229384 2024-11-24T20:47:09,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229384 2024-11-24T20:47:09,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229384) roll requested 2024-11-24T20:47:09,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229392 2024-11-24T20:47:09,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229392 2024-11-24T20:47:09,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229392) roll requested 2024-11-24T20:47:09,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229404 2024-11-24T20:47:09,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229404 2024-11-24T20:47:09,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229404) roll requested 2024-11-24T20:47:09,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229411 2024-11-24T20:47:09,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229411 2024-11-24T20:47:09,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229411) roll requested 2024-11-24T20:47:09,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229420 2024-11-24T20:47:09,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229420 2024-11-24T20:47:09,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229420) roll requested 2024-11-24T20:47:09,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229426 2024-11-24T20:47:09,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229426 2024-11-24T20:47:09,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229426) roll requested 2024-11-24T20:47:09,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229433 2024-11-24T20:47:09,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229433 2024-11-24T20:47:09,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229433) roll requested 2024-11-24T20:47:09,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229439 2024-11-24T20:47:09,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229439 2024-11-24T20:47:09,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229439) roll requested 2024-11-24T20:47:09,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229448 2024-11-24T20:47:09,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229448 2024-11-24T20:47:09,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229448) roll requested 2024-11-24T20:47:09,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229453 2024-11-24T20:47:09,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229453 2024-11-24T20:47:09,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229453) roll requested 2024-11-24T20:47:09,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229462 2024-11-24T20:47:09,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229462 2024-11-24T20:47:09,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229462) roll requested 2024-11-24T20:47:09,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229468 2024-11-24T20:47:09,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229468 2024-11-24T20:47:09,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229468) roll requested 2024-11-24T20:47:09,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229481 2024-11-24T20:47:09,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229481 2024-11-24T20:47:09,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229481) roll requested 2024-11-24T20:47:09,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229487 2024-11-24T20:47:09,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229487 2024-11-24T20:47:09,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229487) roll requested 2024-11-24T20:47:09,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229493 2024-11-24T20:47:09,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229493 2024-11-24T20:47:09,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229493) roll requested 2024-11-24T20:47:09,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229504 2024-11-24T20:47:09,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229504 2024-11-24T20:47:09,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229504) roll requested 2024-11-24T20:47:09,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229512 2024-11-24T20:47:09,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229512 2024-11-24T20:47:09,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229512) roll requested 2024-11-24T20:47:09,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229521 2024-11-24T20:47:09,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229521 2024-11-24T20:47:09,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229521) roll requested 2024-11-24T20:47:09,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229528 2024-11-24T20:47:09,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229528 2024-11-24T20:47:09,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229528) roll requested 2024-11-24T20:47:09,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229536 2024-11-24T20:47:09,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229536 2024-11-24T20:47:09,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229536) roll requested 2024-11-24T20:47:09,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229542 2024-11-24T20:47:09,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229542 2024-11-24T20:47:09,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229542) roll requested 2024-11-24T20:47:09,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229555 2024-11-24T20:47:09,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229555 2024-11-24T20:47:09,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229555) roll requested 2024-11-24T20:47:09,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229565 2024-11-24T20:47:09,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229565 2024-11-24T20:47:09,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229565) roll requested 2024-11-24T20:47:09,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229572 2024-11-24T20:47:09,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229572 2024-11-24T20:47:09,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229572) roll requested 2024-11-24T20:47:09,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229579 2024-11-24T20:47:09,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229579 2024-11-24T20:47:09,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229579) roll requested 2024-11-24T20:47:09,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229584 2024-11-24T20:47:09,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229584 2024-11-24T20:47:09,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229584) roll requested 2024-11-24T20:47:09,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229590 2024-11-24T20:47:09,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229590 2024-11-24T20:47:09,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229590) roll requested 2024-11-24T20:47:09,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229615 2024-11-24T20:47:09,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229615 2024-11-24T20:47:09,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229615) roll requested 2024-11-24T20:47:09,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229629 2024-11-24T20:47:09,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229629 2024-11-24T20:47:09,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229629) roll requested 2024-11-24T20:47:09,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229648 2024-11-24T20:47:09,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229648 2024-11-24T20:47:09,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229648) roll requested 2024-11-24T20:47:09,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229660 2024-11-24T20:47:09,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229660 2024-11-24T20:47:09,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229660) roll requested 2024-11-24T20:47:09,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229668 2024-11-24T20:47:09,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229668 2024-11-24T20:47:09,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229668) roll requested 2024-11-24T20:47:09,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229680 2024-11-24T20:47:09,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229680 2024-11-24T20:47:09,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229680) roll requested 2024-11-24T20:47:09,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229689 2024-11-24T20:47:09,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229689 2024-11-24T20:47:09,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229689) roll requested 2024-11-24T20:47:09,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229697 2024-11-24T20:47:09,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229697 2024-11-24T20:47:09,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229697) roll requested 2024-11-24T20:47:09,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229704 2024-11-24T20:47:09,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229704 2024-11-24T20:47:09,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229704) roll requested 2024-11-24T20:47:09,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229712 2024-11-24T20:47:09,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229712 2024-11-24T20:47:09,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229712) roll requested 2024-11-24T20:47:09,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229726 2024-11-24T20:47:09,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229726 2024-11-24T20:47:09,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229726) roll requested 2024-11-24T20:47:09,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229737 2024-11-24T20:47:09,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229737 2024-11-24T20:47:09,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229737) roll requested 2024-11-24T20:47:09,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229749 2024-11-24T20:47:09,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229749 2024-11-24T20:47:09,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229749) roll requested 2024-11-24T20:47:09,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229756 2024-11-24T20:47:09,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229756 2024-11-24T20:47:09,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229756) roll requested 2024-11-24T20:47:09,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229764 2024-11-24T20:47:09,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229764 2024-11-24T20:47:09,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229764) roll requested 2024-11-24T20:47:09,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229772 2024-11-24T20:47:09,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229772 2024-11-24T20:47:09,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229772) roll requested 2024-11-24T20:47:09,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229780 2024-11-24T20:47:09,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229780 2024-11-24T20:47:09,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229780) roll requested 2024-11-24T20:47:09,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229787 2024-11-24T20:47:09,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229787 2024-11-24T20:47:09,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229787) roll requested 2024-11-24T20:47:09,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229796 2024-11-24T20:47:09,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229796 2024-11-24T20:47:09,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229796) roll requested 2024-11-24T20:47:09,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229804 2024-11-24T20:47:09,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229804 2024-11-24T20:47:09,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229804) roll requested 2024-11-24T20:47:09,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229812 2024-11-24T20:47:09,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229812 2024-11-24T20:47:09,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229812) roll requested 2024-11-24T20:47:09,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229821 2024-11-24T20:47:09,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229821 2024-11-24T20:47:09,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229821) roll requested 2024-11-24T20:47:09,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229833 2024-11-24T20:47:09,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229833 2024-11-24T20:47:09,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229833) roll requested 2024-11-24T20:47:09,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229845 2024-11-24T20:47:09,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229845 2024-11-24T20:47:09,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229845) roll requested 2024-11-24T20:47:09,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229860 2024-11-24T20:47:09,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229860 2024-11-24T20:47:09,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229860) roll requested 2024-11-24T20:47:09,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229872 2024-11-24T20:47:09,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229872 2024-11-24T20:47:09,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229872) roll requested 2024-11-24T20:47:09,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229880 2024-11-24T20:47:09,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229880 2024-11-24T20:47:09,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229880) roll requested 2024-11-24T20:47:09,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229889 2024-11-24T20:47:09,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229889 2024-11-24T20:47:09,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229889) roll requested 2024-11-24T20:47:09,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229896 2024-11-24T20:47:09,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229896 2024-11-24T20:47:09,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229896) roll requested 2024-11-24T20:47:09,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229903 2024-11-24T20:47:09,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229903 2024-11-24T20:47:09,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229903) roll requested 2024-11-24T20:47:09,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229914 2024-11-24T20:47:09,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229914 2024-11-24T20:47:09,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229914) roll requested 2024-11-24T20:47:09,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229923 2024-11-24T20:47:09,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229923 2024-11-24T20:47:09,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229923) roll requested 2024-11-24T20:47:09,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229933 2024-11-24T20:47:09,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229933 2024-11-24T20:47:09,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229933) roll requested 2024-11-24T20:47:09,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229940 2024-11-24T20:47:09,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229940 2024-11-24T20:47:09,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229940) roll requested 2024-11-24T20:47:09,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229949 2024-11-24T20:47:09,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229949 2024-11-24T20:47:09,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229949) roll requested 2024-11-24T20:47:09,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229955 2024-11-24T20:47:09,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229955 2024-11-24T20:47:09,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229955) roll requested 2024-11-24T20:47:09,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229965 2024-11-24T20:47:09,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229965 2024-11-24T20:47:09,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229965) roll requested 2024-11-24T20:47:09,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229982 2024-11-24T20:47:09,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229982 2024-11-24T20:47:09,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229982) roll requested 2024-11-24T20:47:09,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229993 2024-11-24T20:47:09,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:09,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229993 2024-11-24T20:47:09,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:09,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:09,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229993) roll requested 2024-11-24T20:47:09,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481229999 2024-11-24T20:47:10,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229999 2024-11-24T20:47:10,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481229999) roll requested 2024-11-24T20:47:10,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230012 2024-11-24T20:47:10,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481229999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230012 2024-11-24T20:47:10,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230012) roll requested 2024-11-24T20:47:10,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230037 2024-11-24T20:47:10,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230037 2024-11-24T20:47:10,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230037) roll requested 2024-11-24T20:47:10,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230056 2024-11-24T20:47:10,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230056 2024-11-24T20:47:10,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230056) roll requested 2024-11-24T20:47:10,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230073 2024-11-24T20:47:10,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230073 2024-11-24T20:47:10,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230073) roll requested 2024-11-24T20:47:10,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230092 2024-11-24T20:47:10,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230092 2024-11-24T20:47:10,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230092) roll requested 2024-11-24T20:47:10,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230100 2024-11-24T20:47:10,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230100 2024-11-24T20:47:10,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230100) roll requested 2024-11-24T20:47:10,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230108 2024-11-24T20:47:10,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230108 2024-11-24T20:47:10,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230108) roll requested 2024-11-24T20:47:10,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230114 2024-11-24T20:47:10,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230114 2024-11-24T20:47:10,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230114) roll requested 2024-11-24T20:47:10,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230124 2024-11-24T20:47:10,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230124 2024-11-24T20:47:10,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230124) roll requested 2024-11-24T20:47:10,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230131 2024-11-24T20:47:10,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230131 2024-11-24T20:47:10,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230131) roll requested 2024-11-24T20:47:10,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230140 2024-11-24T20:47:10,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230140 2024-11-24T20:47:10,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230140) roll requested 2024-11-24T20:47:10,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230154 2024-11-24T20:47:10,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230154 2024-11-24T20:47:10,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230154) roll requested 2024-11-24T20:47:10,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230164 2024-11-24T20:47:10,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230164 2024-11-24T20:47:10,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230164) roll requested 2024-11-24T20:47:10,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230172 2024-11-24T20:47:10,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230172 2024-11-24T20:47:10,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230172) roll requested 2024-11-24T20:47:10,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230178 2024-11-24T20:47:10,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230178 2024-11-24T20:47:10,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230178) roll requested 2024-11-24T20:47:10,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230184 2024-11-24T20:47:10,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230184 2024-11-24T20:47:10,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230184) roll requested 2024-11-24T20:47:10,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230191 2024-11-24T20:47:10,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230191 2024-11-24T20:47:10,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230191) roll requested 2024-11-24T20:47:10,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230197 2024-11-24T20:47:10,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230197 2024-11-24T20:47:10,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230197) roll requested 2024-11-24T20:47:10,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230204 2024-11-24T20:47:10,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230204 2024-11-24T20:47:10,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230204) roll requested 2024-11-24T20:47:10,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230211 2024-11-24T20:47:10,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230211 2024-11-24T20:47:10,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230211) roll requested 2024-11-24T20:47:10,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230220 2024-11-24T20:47:10,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230220 2024-11-24T20:47:10,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230220) roll requested 2024-11-24T20:47:10,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230228 2024-11-24T20:47:10,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230228 2024-11-24T20:47:10,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230228) roll requested 2024-11-24T20:47:10,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230238 2024-11-24T20:47:10,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230238 2024-11-24T20:47:10,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230238) roll requested 2024-11-24T20:47:10,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230245 2024-11-24T20:47:10,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230245 2024-11-24T20:47:10,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230245) roll requested 2024-11-24T20:47:10,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230251 2024-11-24T20:47:10,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230251 2024-11-24T20:47:10,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230251) roll requested 2024-11-24T20:47:10,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230261 2024-11-24T20:47:10,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230261 2024-11-24T20:47:10,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230261) roll requested 2024-11-24T20:47:10,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230284 2024-11-24T20:47:10,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230284 2024-11-24T20:47:10,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230284) roll requested 2024-11-24T20:47:10,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230292 2024-11-24T20:47:10,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230292 2024-11-24T20:47:10,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230292) roll requested 2024-11-24T20:47:10,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230298 2024-11-24T20:47:10,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230298 2024-11-24T20:47:10,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230298) roll requested 2024-11-24T20:47:10,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230308 2024-11-24T20:47:10,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230308 2024-11-24T20:47:10,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230308) roll requested 2024-11-24T20:47:10,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230318 2024-11-24T20:47:10,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230318 2024-11-24T20:47:10,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230318) roll requested 2024-11-24T20:47:10,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230330 2024-11-24T20:47:10,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230330 2024-11-24T20:47:10,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230330) roll requested 2024-11-24T20:47:10,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230340 2024-11-24T20:47:10,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230340 2024-11-24T20:47:10,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230340) roll requested 2024-11-24T20:47:10,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230347 2024-11-24T20:47:10,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230347 2024-11-24T20:47:10,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230347) roll requested 2024-11-24T20:47:10,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230357 2024-11-24T20:47:10,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230357 2024-11-24T20:47:10,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230357) roll requested 2024-11-24T20:47:10,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230365 2024-11-24T20:47:10,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230365 2024-11-24T20:47:10,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230365) roll requested 2024-11-24T20:47:10,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230372 2024-11-24T20:47:10,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230372 2024-11-24T20:47:10,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230372) roll requested 2024-11-24T20:47:10,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230378 2024-11-24T20:47:10,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230378 2024-11-24T20:47:10,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230378) roll requested 2024-11-24T20:47:10,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230384 2024-11-24T20:47:10,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230384 2024-11-24T20:47:10,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230384) roll requested 2024-11-24T20:47:10,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230392 2024-11-24T20:47:10,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230392 2024-11-24T20:47:10,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230392) roll requested 2024-11-24T20:47:10,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230399 2024-11-24T20:47:10,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230399 2024-11-24T20:47:10,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230399) roll requested 2024-11-24T20:47:10,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230406 2024-11-24T20:47:10,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230406 2024-11-24T20:47:10,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230406) roll requested 2024-11-24T20:47:10,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230412 2024-11-24T20:47:10,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230412 2024-11-24T20:47:10,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230412) roll requested 2024-11-24T20:47:10,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230421 2024-11-24T20:47:10,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230421 2024-11-24T20:47:10,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230421) roll requested 2024-11-24T20:47:10,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230428 2024-11-24T20:47:10,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230428 2024-11-24T20:47:10,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230428) roll requested 2024-11-24T20:47:10,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230439 2024-11-24T20:47:10,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230439 2024-11-24T20:47:10,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230439) roll requested 2024-11-24T20:47:10,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230449 2024-11-24T20:47:10,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230449 2024-11-24T20:47:10,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230449) roll requested 2024-11-24T20:47:10,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230457 2024-11-24T20:47:10,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230457 2024-11-24T20:47:10,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230457) roll requested 2024-11-24T20:47:10,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230467 2024-11-24T20:47:10,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230467 2024-11-24T20:47:10,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230467) roll requested 2024-11-24T20:47:10,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230476 2024-11-24T20:47:10,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230476 2024-11-24T20:47:10,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230476) roll requested 2024-11-24T20:47:10,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230527 2024-11-24T20:47:10,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230527 2024-11-24T20:47:10,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230527) roll requested 2024-11-24T20:47:10,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230545 2024-11-24T20:47:10,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230545 2024-11-24T20:47:10,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230545) roll requested 2024-11-24T20:47:10,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230572 2024-11-24T20:47:10,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230572 2024-11-24T20:47:10,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230572) roll requested 2024-11-24T20:47:10,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230579 2024-11-24T20:47:10,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230579 2024-11-24T20:47:10,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230579) roll requested 2024-11-24T20:47:10,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230585 2024-11-24T20:47:10,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230585 2024-11-24T20:47:10,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230585) roll requested 2024-11-24T20:47:10,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230593 2024-11-24T20:47:10,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230593 2024-11-24T20:47:10,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230593) roll requested 2024-11-24T20:47:10,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230602 2024-11-24T20:47:10,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230602 2024-11-24T20:47:10,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230602) roll requested 2024-11-24T20:47:10,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230608 2024-11-24T20:47:10,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230608 2024-11-24T20:47:10,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230608) roll requested 2024-11-24T20:47:10,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230615 2024-11-24T20:47:10,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230615 2024-11-24T20:47:10,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230615) roll requested 2024-11-24T20:47:10,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230620 2024-11-24T20:47:10,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230620 2024-11-24T20:47:10,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230620) roll requested 2024-11-24T20:47:10,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230627 2024-11-24T20:47:10,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230627 2024-11-24T20:47:10,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230627) roll requested 2024-11-24T20:47:10,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230635 2024-11-24T20:47:10,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230635 2024-11-24T20:47:10,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230635) roll requested 2024-11-24T20:47:10,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230646 2024-11-24T20:47:10,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230646 2024-11-24T20:47:10,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230646) roll requested 2024-11-24T20:47:10,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230675 2024-11-24T20:47:10,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230675 2024-11-24T20:47:10,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230675) roll requested 2024-11-24T20:47:10,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230700 2024-11-24T20:47:10,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230700 2024-11-24T20:47:10,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230700) roll requested 2024-11-24T20:47:10,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230713 2024-11-24T20:47:10,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230713 2024-11-24T20:47:10,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230713) roll requested 2024-11-24T20:47:10,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230741 2024-11-24T20:47:10,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230741 2024-11-24T20:47:10,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230741) roll requested 2024-11-24T20:47:10,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230758 2024-11-24T20:47:10,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230758 2024-11-24T20:47:10,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230758) roll requested 2024-11-24T20:47:10,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230768 2024-11-24T20:47:10,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230768 2024-11-24T20:47:10,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230768) roll requested 2024-11-24T20:47:10,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230781 2024-11-24T20:47:10,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230781 2024-11-24T20:47:10,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230781) roll requested 2024-11-24T20:47:10,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230800 2024-11-24T20:47:10,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230800 2024-11-24T20:47:10,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230800) roll requested 2024-11-24T20:47:10,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230808 2024-11-24T20:47:10,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230808 2024-11-24T20:47:10,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230808) roll requested 2024-11-24T20:47:10,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230821 2024-11-24T20:47:10,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230821 2024-11-24T20:47:10,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230821) roll requested 2024-11-24T20:47:10,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230827 2024-11-24T20:47:10,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230827 2024-11-24T20:47:10,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230827) roll requested 2024-11-24T20:47:10,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230837 2024-11-24T20:47:10,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230837 2024-11-24T20:47:10,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230837) roll requested 2024-11-24T20:47:10,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230846 2024-11-24T20:47:10,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230846 2024-11-24T20:47:10,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230846) roll requested 2024-11-24T20:47:10,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230900 2024-11-24T20:47:10,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230900 2024-11-24T20:47:10,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230900) roll requested 2024-11-24T20:47:10,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230911 2024-11-24T20:47:10,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230911 2024-11-24T20:47:10,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230911) roll requested 2024-11-24T20:47:10,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230924 2024-11-24T20:47:10,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230924 2024-11-24T20:47:10,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230924) roll requested 2024-11-24T20:47:10,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230933 2024-11-24T20:47:10,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230933 2024-11-24T20:47:10,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230933) roll requested 2024-11-24T20:47:10,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230940 2024-11-24T20:47:10,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230940 2024-11-24T20:47:10,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230940) roll requested 2024-11-24T20:47:10,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230953 2024-11-24T20:47:10,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230953 2024-11-24T20:47:10,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230953) roll requested 2024-11-24T20:47:10,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230964 2024-11-24T20:47:10,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230964 2024-11-24T20:47:10,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230964) roll requested 2024-11-24T20:47:10,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230973 2024-11-24T20:47:10,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230973 2024-11-24T20:47:10,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230973) roll requested 2024-11-24T20:47:10,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230980 2024-11-24T20:47:10,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230980 2024-11-24T20:47:10,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:10,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:10,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230980) roll requested 2024-11-24T20:47:10,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481230992 2024-11-24T20:47:10,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:10,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230992 2024-11-24T20:47:11,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481230992) roll requested 2024-11-24T20:47:11,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231000 2024-11-24T20:47:11,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481230992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231000 2024-11-24T20:47:11,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231000) roll requested 2024-11-24T20:47:11,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231007 2024-11-24T20:47:11,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231007 2024-11-24T20:47:11,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231007) roll requested 2024-11-24T20:47:11,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231013 2024-11-24T20:47:11,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231013 2024-11-24T20:47:11,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231013) roll requested 2024-11-24T20:47:11,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231020 2024-11-24T20:47:11,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231020 2024-11-24T20:47:11,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231020) roll requested 2024-11-24T20:47:11,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231032 2024-11-24T20:47:11,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231032 2024-11-24T20:47:11,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231032) roll requested 2024-11-24T20:47:11,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231057 2024-11-24T20:47:11,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231057 2024-11-24T20:47:11,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231057) roll requested 2024-11-24T20:47:11,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231077 2024-11-24T20:47:11,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231077 2024-11-24T20:47:11,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231077) roll requested 2024-11-24T20:47:11,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231088 2024-11-24T20:47:11,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231088 2024-11-24T20:47:11,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231088) roll requested 2024-11-24T20:47:11,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231095 2024-11-24T20:47:11,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231095 2024-11-24T20:47:11,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231095) roll requested 2024-11-24T20:47:11,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231105 2024-11-24T20:47:11,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231105 2024-11-24T20:47:11,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231105) roll requested 2024-11-24T20:47:11,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231111 2024-11-24T20:47:11,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231111 2024-11-24T20:47:11,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231111) roll requested 2024-11-24T20:47:11,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231120 2024-11-24T20:47:11,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231120 2024-11-24T20:47:11,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231120) roll requested 2024-11-24T20:47:11,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231128 2024-11-24T20:47:11,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231128 2024-11-24T20:47:11,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231128) roll requested 2024-11-24T20:47:11,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231149 2024-11-24T20:47:11,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231149 2024-11-24T20:47:11,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231149) roll requested 2024-11-24T20:47:11,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231156 2024-11-24T20:47:11,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231156 2024-11-24T20:47:11,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231156) roll requested 2024-11-24T20:47:11,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231164 2024-11-24T20:47:11,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231164 2024-11-24T20:47:11,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231164) roll requested 2024-11-24T20:47:11,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231176 2024-11-24T20:47:11,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231176 2024-11-24T20:47:11,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231176) roll requested 2024-11-24T20:47:11,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231183 2024-11-24T20:47:11,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231183 2024-11-24T20:47:11,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231183) roll requested 2024-11-24T20:47:11,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231200 2024-11-24T20:47:11,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231200 2024-11-24T20:47:11,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231200) roll requested 2024-11-24T20:47:11,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231216 2024-11-24T20:47:11,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231216 2024-11-24T20:47:11,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231216) roll requested 2024-11-24T20:47:11,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231224 2024-11-24T20:47:11,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231224 2024-11-24T20:47:11,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231224) roll requested 2024-11-24T20:47:11,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231233 2024-11-24T20:47:11,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231233 2024-11-24T20:47:11,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231233) roll requested 2024-11-24T20:47:11,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231240 2024-11-24T20:47:11,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231240 2024-11-24T20:47:11,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231240) roll requested 2024-11-24T20:47:11,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231251 2024-11-24T20:47:11,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231251 2024-11-24T20:47:11,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231251) roll requested 2024-11-24T20:47:11,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231259 2024-11-24T20:47:11,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231259 2024-11-24T20:47:11,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231259) roll requested 2024-11-24T20:47:11,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231273 2024-11-24T20:47:11,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231273 2024-11-24T20:47:11,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231273) roll requested 2024-11-24T20:47:11,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231281 2024-11-24T20:47:11,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231281 2024-11-24T20:47:11,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231281) roll requested 2024-11-24T20:47:11,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231292 2024-11-24T20:47:11,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231292 2024-11-24T20:47:11,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231292) roll requested 2024-11-24T20:47:11,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231298 2024-11-24T20:47:11,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231298 2024-11-24T20:47:11,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231298) roll requested 2024-11-24T20:47:11,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231312 2024-11-24T20:47:11,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231312 2024-11-24T20:47:11,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231312) roll requested 2024-11-24T20:47:11,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231321 2024-11-24T20:47:11,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231321 2024-11-24T20:47:11,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231321) roll requested 2024-11-24T20:47:11,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231333 2024-11-24T20:47:11,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231333 2024-11-24T20:47:11,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231333) roll requested 2024-11-24T20:47:11,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231348 2024-11-24T20:47:11,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231348 2024-11-24T20:47:11,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231348) roll requested 2024-11-24T20:47:11,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231360 2024-11-24T20:47:11,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231360 2024-11-24T20:47:11,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231360) roll requested 2024-11-24T20:47:11,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231372 2024-11-24T20:47:11,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231372 2024-11-24T20:47:11,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231372) roll requested 2024-11-24T20:47:11,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231384 2024-11-24T20:47:11,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231384 2024-11-24T20:47:11,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231384) roll requested 2024-11-24T20:47:11,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231392 2024-11-24T20:47:11,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231392 2024-11-24T20:47:11,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231392) roll requested 2024-11-24T20:47:11,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231401 2024-11-24T20:47:11,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231401 2024-11-24T20:47:11,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231401) roll requested 2024-11-24T20:47:11,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231407 2024-11-24T20:47:11,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231407 2024-11-24T20:47:11,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231407) roll requested 2024-11-24T20:47:11,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231416 2024-11-24T20:47:11,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231416 2024-11-24T20:47:11,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231416) roll requested 2024-11-24T20:47:11,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231423 2024-11-24T20:47:11,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231423 2024-11-24T20:47:11,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231423) roll requested 2024-11-24T20:47:11,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231435 2024-11-24T20:47:11,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231435 2024-11-24T20:47:11,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231435) roll requested 2024-11-24T20:47:11,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231441 2024-11-24T20:47:11,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231441 2024-11-24T20:47:11,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231441) roll requested 2024-11-24T20:47:11,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231448 2024-11-24T20:47:11,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231448 2024-11-24T20:47:11,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231448) roll requested 2024-11-24T20:47:11,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231455 2024-11-24T20:47:11,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231455 2024-11-24T20:47:11,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231455) roll requested 2024-11-24T20:47:11,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231473 2024-11-24T20:47:11,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231473 2024-11-24T20:47:11,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231473) roll requested 2024-11-24T20:47:11,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231480 2024-11-24T20:47:11,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231480 2024-11-24T20:47:11,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231480) roll requested 2024-11-24T20:47:11,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231488 2024-11-24T20:47:11,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231488 2024-11-24T20:47:11,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231488) roll requested 2024-11-24T20:47:11,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231505 2024-11-24T20:47:11,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231505 2024-11-24T20:47:11,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231505) roll requested 2024-11-24T20:47:11,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231517 2024-11-24T20:47:11,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231517 2024-11-24T20:47:11,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231517) roll requested 2024-11-24T20:47:11,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231529 2024-11-24T20:47:11,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231529 2024-11-24T20:47:11,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231529) roll requested 2024-11-24T20:47:11,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231541 2024-11-24T20:47:11,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231541 2024-11-24T20:47:11,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231541) roll requested 2024-11-24T20:47:11,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231548 2024-11-24T20:47:11,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231548 2024-11-24T20:47:11,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231548) roll requested 2024-11-24T20:47:11,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231555 2024-11-24T20:47:11,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231555 2024-11-24T20:47:11,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231555) roll requested 2024-11-24T20:47:11,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231564 2024-11-24T20:47:11,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231564 2024-11-24T20:47:11,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231564) roll requested 2024-11-24T20:47:11,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231571 2024-11-24T20:47:11,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231571 2024-11-24T20:47:11,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231571) roll requested 2024-11-24T20:47:11,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231577 2024-11-24T20:47:11,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231577 2024-11-24T20:47:11,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231577) roll requested 2024-11-24T20:47:11,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231584 2024-11-24T20:47:11,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231584 2024-11-24T20:47:11,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231584) roll requested 2024-11-24T20:47:11,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231590 2024-11-24T20:47:11,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231590 2024-11-24T20:47:11,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231590) roll requested 2024-11-24T20:47:11,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231600 2024-11-24T20:47:11,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231600 2024-11-24T20:47:11,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231600) roll requested 2024-11-24T20:47:11,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231610 2024-11-24T20:47:11,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231610 2024-11-24T20:47:11,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231610) roll requested 2024-11-24T20:47:11,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231619 2024-11-24T20:47:11,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231619 2024-11-24T20:47:11,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231619) roll requested 2024-11-24T20:47:11,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231628 2024-11-24T20:47:11,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231628 2024-11-24T20:47:11,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231628) roll requested 2024-11-24T20:47:11,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231634 2024-11-24T20:47:11,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231634 2024-11-24T20:47:11,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231634) roll requested 2024-11-24T20:47:11,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231640 2024-11-24T20:47:11,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231640 2024-11-24T20:47:11,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231640) roll requested 2024-11-24T20:47:11,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231647 2024-11-24T20:47:11,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231647 2024-11-24T20:47:11,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231647) roll requested 2024-11-24T20:47:11,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231653 2024-11-24T20:47:11,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231653 2024-11-24T20:47:11,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231653) roll requested 2024-11-24T20:47:11,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231660 2024-11-24T20:47:11,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231660 2024-11-24T20:47:11,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231660) roll requested 2024-11-24T20:47:11,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231669 2024-11-24T20:47:11,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231669 2024-11-24T20:47:11,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231669) roll requested 2024-11-24T20:47:11,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231676 2024-11-24T20:47:11,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231676 2024-11-24T20:47:11,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231676) roll requested 2024-11-24T20:47:11,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231688 2024-11-24T20:47:11,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231688 2024-11-24T20:47:11,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231688) roll requested 2024-11-24T20:47:11,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231704 2024-11-24T20:47:11,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231704 2024-11-24T20:47:11,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231704) roll requested 2024-11-24T20:47:11,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231725 2024-11-24T20:47:11,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231725 2024-11-24T20:47:11,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231725) roll requested 2024-11-24T20:47:11,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231733 2024-11-24T20:47:11,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231733 2024-11-24T20:47:11,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231733) roll requested 2024-11-24T20:47:11,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231753 2024-11-24T20:47:11,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231753 2024-11-24T20:47:11,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231753) roll requested 2024-11-24T20:47:11,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231764 2024-11-24T20:47:11,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231764 2024-11-24T20:47:11,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231764) roll requested 2024-11-24T20:47:11,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231777 2024-11-24T20:47:11,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231777 2024-11-24T20:47:11,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231777) roll requested 2024-11-24T20:47:11,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231784 2024-11-24T20:47:11,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231784 2024-11-24T20:47:11,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231784) roll requested 2024-11-24T20:47:11,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231796 2024-11-24T20:47:11,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231796 2024-11-24T20:47:11,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231796) roll requested 2024-11-24T20:47:11,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231808 2024-11-24T20:47:11,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231808 2024-11-24T20:47:11,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231808) roll requested 2024-11-24T20:47:11,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231819 2024-11-24T20:47:11,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231819 2024-11-24T20:47:11,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231819) roll requested 2024-11-24T20:47:11,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231830 2024-11-24T20:47:11,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231830 2024-11-24T20:47:11,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231830) roll requested 2024-11-24T20:47:11,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231840 2024-11-24T20:47:11,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231840 2024-11-24T20:47:11,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231840) roll requested 2024-11-24T20:47:11,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231853 2024-11-24T20:47:11,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231853 2024-11-24T20:47:11,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231853) roll requested 2024-11-24T20:47:11,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231861 2024-11-24T20:47:11,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231861 2024-11-24T20:47:11,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231861) roll requested 2024-11-24T20:47:11,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231878 2024-11-24T20:47:11,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231878 2024-11-24T20:47:11,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231878) roll requested 2024-11-24T20:47:11,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231889 2024-11-24T20:47:11,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231889 2024-11-24T20:47:11,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231889) roll requested 2024-11-24T20:47:11,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231901 2024-11-24T20:47:11,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231901 2024-11-24T20:47:11,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231901) roll requested 2024-11-24T20:47:11,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231908 2024-11-24T20:47:11,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231908 2024-11-24T20:47:11,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231908) roll requested 2024-11-24T20:47:11,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231916 2024-11-24T20:47:11,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231916 2024-11-24T20:47:11,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231916) roll requested 2024-11-24T20:47:11,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231924 2024-11-24T20:47:11,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231924 2024-11-24T20:47:11,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231924) roll requested 2024-11-24T20:47:11,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231930 2024-11-24T20:47:11,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231930 2024-11-24T20:47:11,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231930) roll requested 2024-11-24T20:47:11,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231936 2024-11-24T20:47:11,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231936 2024-11-24T20:47:11,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231936) roll requested 2024-11-24T20:47:11,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231943 2024-11-24T20:47:11,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231943 2024-11-24T20:47:11,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231943) roll requested 2024-11-24T20:47:11,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231950 2024-11-24T20:47:11,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231950 2024-11-24T20:47:11,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231950) roll requested 2024-11-24T20:47:11,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231958 2024-11-24T20:47:11,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231958 2024-11-24T20:47:11,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231958) roll requested 2024-11-24T20:47:11,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231965 2024-11-24T20:47:11,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231965 2024-11-24T20:47:11,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231965) roll requested 2024-11-24T20:47:11,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231973 2024-11-24T20:47:11,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231973 2024-11-24T20:47:11,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231973) roll requested 2024-11-24T20:47:11,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231980 2024-11-24T20:47:11,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231980 2024-11-24T20:47:11,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231980) roll requested 2024-11-24T20:47:11,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231986 2024-11-24T20:47:11,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:11,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231986 2024-11-24T20:47:11,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:11,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:11,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231986) roll requested 2024-11-24T20:47:11,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481231994 2024-11-24T20:47:12,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231994 2024-11-24T20:47:12,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481231994) roll requested 2024-11-24T20:47:12,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232001 2024-11-24T20:47:12,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481231994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232001 2024-11-24T20:47:12,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232001) roll requested 2024-11-24T20:47:12,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232008 2024-11-24T20:47:12,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232008 2024-11-24T20:47:12,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232008) roll requested 2024-11-24T20:47:12,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232015 2024-11-24T20:47:12,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232015 2024-11-24T20:47:12,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232015) roll requested 2024-11-24T20:47:12,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232021 2024-11-24T20:47:12,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232021 2024-11-24T20:47:12,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232021) roll requested 2024-11-24T20:47:12,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232028 2024-11-24T20:47:12,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232028 2024-11-24T20:47:12,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232028) roll requested 2024-11-24T20:47:12,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232035 2024-11-24T20:47:12,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232035 2024-11-24T20:47:12,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232035) roll requested 2024-11-24T20:47:12,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232046 2024-11-24T20:47:12,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232046 2024-11-24T20:47:12,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232046) roll requested 2024-11-24T20:47:12,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232053 2024-11-24T20:47:12,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232053 2024-11-24T20:47:12,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232053) roll requested 2024-11-24T20:47:12,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232066 2024-11-24T20:47:12,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232066 2024-11-24T20:47:12,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232066) roll requested 2024-11-24T20:47:12,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232085 2024-11-24T20:47:12,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232085 2024-11-24T20:47:12,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232085) roll requested 2024-11-24T20:47:12,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232093 2024-11-24T20:47:12,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232093 2024-11-24T20:47:12,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232093) roll requested 2024-11-24T20:47:12,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232115 2024-11-24T20:47:12,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232115 2024-11-24T20:47:12,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232115) roll requested 2024-11-24T20:47:12,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232124 2024-11-24T20:47:12,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232124 2024-11-24T20:47:12,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232124) roll requested 2024-11-24T20:47:12,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232133 2024-11-24T20:47:12,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232133 2024-11-24T20:47:12,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232133) roll requested 2024-11-24T20:47:12,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232140 2024-11-24T20:47:12,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232140 2024-11-24T20:47:12,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232140) roll requested 2024-11-24T20:47:12,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232148 2024-11-24T20:47:12,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232148 2024-11-24T20:47:12,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232148) roll requested 2024-11-24T20:47:12,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232183 2024-11-24T20:47:12,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232183 2024-11-24T20:47:12,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232183) roll requested 2024-11-24T20:47:12,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232205 2024-11-24T20:47:12,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232205 2024-11-24T20:47:12,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232205) roll requested 2024-11-24T20:47:12,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232220 2024-11-24T20:47:12,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232220 2024-11-24T20:47:12,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232220) roll requested 2024-11-24T20:47:12,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232234 2024-11-24T20:47:12,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232234 2024-11-24T20:47:12,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232234) roll requested 2024-11-24T20:47:12,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232244 2024-11-24T20:47:12,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232244 2024-11-24T20:47:12,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232244) roll requested 2024-11-24T20:47:12,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232254 2024-11-24T20:47:12,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232254 2024-11-24T20:47:12,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232254) roll requested 2024-11-24T20:47:12,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232265 2024-11-24T20:47:12,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232265 2024-11-24T20:47:12,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232265) roll requested 2024-11-24T20:47:12,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232277 2024-11-24T20:47:12,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232277 2024-11-24T20:47:12,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232277) roll requested 2024-11-24T20:47:12,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232284 2024-11-24T20:47:12,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232284 2024-11-24T20:47:12,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232284) roll requested 2024-11-24T20:47:12,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232296 2024-11-24T20:47:12,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232296 2024-11-24T20:47:12,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232296) roll requested 2024-11-24T20:47:12,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232309 2024-11-24T20:47:12,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232309 2024-11-24T20:47:12,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232309) roll requested 2024-11-24T20:47:12,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232318 2024-11-24T20:47:12,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232318 2024-11-24T20:47:12,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232318) roll requested 2024-11-24T20:47:12,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232327 2024-11-24T20:47:12,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232327 2024-11-24T20:47:12,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232327) roll requested 2024-11-24T20:47:12,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232335 2024-11-24T20:47:12,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232335 2024-11-24T20:47:12,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232335) roll requested 2024-11-24T20:47:12,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232344 2024-11-24T20:47:12,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232344 2024-11-24T20:47:12,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232344) roll requested 2024-11-24T20:47:12,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232357 2024-11-24T20:47:12,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232357 2024-11-24T20:47:12,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232357) roll requested 2024-11-24T20:47:12,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232369 2024-11-24T20:47:12,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232369 2024-11-24T20:47:12,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232369) roll requested 2024-11-24T20:47:12,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232388 2024-11-24T20:47:12,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232388 2024-11-24T20:47:12,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232388) roll requested 2024-11-24T20:47:12,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232409 2024-11-24T20:47:12,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232409 2024-11-24T20:47:12,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232409) roll requested 2024-11-24T20:47:12,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232424 2024-11-24T20:47:12,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232424 2024-11-24T20:47:12,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232424) roll requested 2024-11-24T20:47:12,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232440 2024-11-24T20:47:12,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232440 2024-11-24T20:47:12,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232440) roll requested 2024-11-24T20:47:12,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232453 2024-11-24T20:47:12,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232453 2024-11-24T20:47:12,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232453) roll requested 2024-11-24T20:47:12,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232461 2024-11-24T20:47:12,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232461 2024-11-24T20:47:12,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232453 is not closed yet, will try archiving it next time 2024-11-24T20:47:12,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232461) roll requested 2024-11-24T20:47:12,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232469 2024-11-24T20:47:12,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232469 2024-11-24T20:47:12,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232469) roll requested 2024-11-24T20:47:12,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232492 2024-11-24T20:47:12,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232492 2024-11-24T20:47:12,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232492) roll requested 2024-11-24T20:47:12,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232516 2024-11-24T20:47:12,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232516 2024-11-24T20:47:12,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232516) roll requested 2024-11-24T20:47:12,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232536 2024-11-24T20:47:12,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232536 2024-11-24T20:47:12,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232536) roll requested 2024-11-24T20:47:12,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232549 2024-11-24T20:47:12,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232549 2024-11-24T20:47:12,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232549) roll requested 2024-11-24T20:47:12,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232565 2024-11-24T20:47:12,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232565 2024-11-24T20:47:12,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232565) roll requested 2024-11-24T20:47:12,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232577 2024-11-24T20:47:12,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232577 2024-11-24T20:47:12,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232577) roll requested 2024-11-24T20:47:12,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232588 2024-11-24T20:47:12,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232588 2024-11-24T20:47:12,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232588) roll requested 2024-11-24T20:47:12,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232595 2024-11-24T20:47:12,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232595 2024-11-24T20:47:12,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232595) roll requested 2024-11-24T20:47:12,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232608 2024-11-24T20:47:12,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232608 2024-11-24T20:47:12,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232608) roll requested 2024-11-24T20:47:12,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232617 2024-11-24T20:47:12,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232617 2024-11-24T20:47:12,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232617) roll requested 2024-11-24T20:47:12,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232632 2024-11-24T20:47:12,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232632 2024-11-24T20:47:12,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232632) roll requested 2024-11-24T20:47:12,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232638 2024-11-24T20:47:12,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232638 2024-11-24T20:47:12,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232638) roll requested 2024-11-24T20:47:12,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232652 2024-11-24T20:47:12,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232652 2024-11-24T20:47:12,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232652) roll requested 2024-11-24T20:47:12,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232660 2024-11-24T20:47:12,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232660 2024-11-24T20:47:12,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232660) roll requested 2024-11-24T20:47:12,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232672 2024-11-24T20:47:12,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232672 2024-11-24T20:47:12,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232672) roll requested 2024-11-24T20:47:12,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232689 2024-11-24T20:47:12,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232689 2024-11-24T20:47:12,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232689) roll requested 2024-11-24T20:47:12,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232706 2024-11-24T20:47:12,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232706 2024-11-24T20:47:12,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232706) roll requested 2024-11-24T20:47:12,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232719 2024-11-24T20:47:12,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232719 2024-11-24T20:47:12,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232719) roll requested 2024-11-24T20:47:12,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232728 2024-11-24T20:47:12,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232728 2024-11-24T20:47:12,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232728) roll requested 2024-11-24T20:47:12,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232736 2024-11-24T20:47:12,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232736 2024-11-24T20:47:12,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232736) roll requested 2024-11-24T20:47:12,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232748 2024-11-24T20:47:12,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232748 2024-11-24T20:47:12,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232748) roll requested 2024-11-24T20:47:12,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232760 2024-11-24T20:47:12,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232760 2024-11-24T20:47:12,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232760) roll requested 2024-11-24T20:47:12,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232768 2024-11-24T20:47:12,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232768 2024-11-24T20:47:12,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232768) roll requested 2024-11-24T20:47:12,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232776 2024-11-24T20:47:12,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232776 2024-11-24T20:47:12,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232776) roll requested 2024-11-24T20:47:12,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232785 2024-11-24T20:47:12,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232785 2024-11-24T20:47:12,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232785) roll requested 2024-11-24T20:47:12,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232804 2024-11-24T20:47:12,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232804 2024-11-24T20:47:12,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232804) roll requested 2024-11-24T20:47:12,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232816 2024-11-24T20:47:12,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232816 2024-11-24T20:47:12,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232816) roll requested 2024-11-24T20:47:12,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232828 2024-11-24T20:47:12,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232828 2024-11-24T20:47:12,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232828) roll requested 2024-11-24T20:47:12,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232837 2024-11-24T20:47:12,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232837 2024-11-24T20:47:12,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232837) roll requested 2024-11-24T20:47:12,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232845 2024-11-24T20:47:12,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232845 2024-11-24T20:47:12,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232845) roll requested 2024-11-24T20:47:12,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232853 2024-11-24T20:47:12,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232853 2024-11-24T20:47:12,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232853) roll requested 2024-11-24T20:47:12,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232865 2024-11-24T20:47:12,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232865 2024-11-24T20:47:12,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232865) roll requested 2024-11-24T20:47:12,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232874 2024-11-24T20:47:12,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232874 2024-11-24T20:47:12,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232874) roll requested 2024-11-24T20:47:12,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232880 2024-11-24T20:47:12,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232880 2024-11-24T20:47:12,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232880) roll requested 2024-11-24T20:47:12,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232913 2024-11-24T20:47:12,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232913 2024-11-24T20:47:12,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232913) roll requested 2024-11-24T20:47:12,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232940 2024-11-24T20:47:12,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232940 2024-11-24T20:47:12,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232940) roll requested 2024-11-24T20:47:12,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232968 2024-11-24T20:47:12,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232968 2024-11-24T20:47:12,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:12,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:12,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232968) roll requested 2024-11-24T20:47:12,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481232981 2024-11-24T20:47:12,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:12,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232981 2024-11-24T20:47:13,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481232981) roll requested 2024-11-24T20:47:13,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233004 2024-11-24T20:47:13,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481232981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233004 2024-11-24T20:47:13,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233004) roll requested 2024-11-24T20:47:13,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233016 2024-11-24T20:47:13,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233016 2024-11-24T20:47:13,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233016) roll requested 2024-11-24T20:47:13,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233029 2024-11-24T20:47:13,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233029 2024-11-24T20:47:13,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233029) roll requested 2024-11-24T20:47:13,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233045 2024-11-24T20:47:13,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233045 2024-11-24T20:47:13,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233045) roll requested 2024-11-24T20:47:13,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233056 2024-11-24T20:47:13,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233056 2024-11-24T20:47:13,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233056) roll requested 2024-11-24T20:47:13,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233070 2024-11-24T20:47:13,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233070 2024-11-24T20:47:13,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233070) roll requested 2024-11-24T20:47:13,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233091 2024-11-24T20:47:13,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233091 2024-11-24T20:47:13,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233091) roll requested 2024-11-24T20:47:13,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233105 2024-11-24T20:47:13,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233105 2024-11-24T20:47:13,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233105) roll requested 2024-11-24T20:47:13,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233116 2024-11-24T20:47:13,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233116 2024-11-24T20:47:13,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233116) roll requested 2024-11-24T20:47:13,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233125 2024-11-24T20:47:13,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233125 2024-11-24T20:47:13,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233125) roll requested 2024-11-24T20:47:13,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233132 2024-11-24T20:47:13,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233132 2024-11-24T20:47:13,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233132) roll requested 2024-11-24T20:47:13,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233137 2024-11-24T20:47:13,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233137 2024-11-24T20:47:13,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233137) roll requested 2024-11-24T20:47:13,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233148 2024-11-24T20:47:13,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233148 2024-11-24T20:47:13,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233148) roll requested 2024-11-24T20:47:13,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233155 2024-11-24T20:47:13,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233155 2024-11-24T20:47:13,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233155) roll requested 2024-11-24T20:47:13,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233164 2024-11-24T20:47:13,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233164 2024-11-24T20:47:13,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233164) roll requested 2024-11-24T20:47:13,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233172 2024-11-24T20:47:13,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233172 2024-11-24T20:47:13,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233172) roll requested 2024-11-24T20:47:13,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233187 2024-11-24T20:47:13,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233187 2024-11-24T20:47:13,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233187) roll requested 2024-11-24T20:47:13,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233196 2024-11-24T20:47:13,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233196 2024-11-24T20:47:13,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233196) roll requested 2024-11-24T20:47:13,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233201 2024-11-24T20:47:13,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233201 2024-11-24T20:47:13,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233201) roll requested 2024-11-24T20:47:13,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233220 2024-11-24T20:47:13,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233220 2024-11-24T20:47:13,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233220) roll requested 2024-11-24T20:47:13,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233236 2024-11-24T20:47:13,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233236 2024-11-24T20:47:13,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233236) roll requested 2024-11-24T20:47:13,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233253 2024-11-24T20:47:13,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233253 2024-11-24T20:47:13,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233253) roll requested 2024-11-24T20:47:13,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233265 2024-11-24T20:47:13,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233265 2024-11-24T20:47:13,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233265) roll requested 2024-11-24T20:47:13,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233272 2024-11-24T20:47:13,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233272 2024-11-24T20:47:13,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233272) roll requested 2024-11-24T20:47:13,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233281 2024-11-24T20:47:13,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233281 2024-11-24T20:47:13,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233281) roll requested 2024-11-24T20:47:13,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233288 2024-11-24T20:47:13,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233288 2024-11-24T20:47:13,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233288) roll requested 2024-11-24T20:47:13,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233297 2024-11-24T20:47:13,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233297 2024-11-24T20:47:13,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233297) roll requested 2024-11-24T20:47:13,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233308 2024-11-24T20:47:13,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233308 2024-11-24T20:47:13,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233308) roll requested 2024-11-24T20:47:13,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233326 2024-11-24T20:47:13,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233326 2024-11-24T20:47:13,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233326) roll requested 2024-11-24T20:47:13,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233336 2024-11-24T20:47:13,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233336 2024-11-24T20:47:13,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233336) roll requested 2024-11-24T20:47:13,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233348 2024-11-24T20:47:13,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233348 2024-11-24T20:47:13,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233348) roll requested 2024-11-24T20:47:13,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233360 2024-11-24T20:47:13,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233360 2024-11-24T20:47:13,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233360) roll requested 2024-11-24T20:47:13,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233368 2024-11-24T20:47:13,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233368 2024-11-24T20:47:13,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233368) roll requested 2024-11-24T20:47:13,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233383 2024-11-24T20:47:13,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233383 2024-11-24T20:47:13,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233383) roll requested 2024-11-24T20:47:13,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233389 2024-11-24T20:47:13,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233389 2024-11-24T20:47:13,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233389) roll requested 2024-11-24T20:47:13,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233397 2024-11-24T20:47:13,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233397 2024-11-24T20:47:13,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233397) roll requested 2024-11-24T20:47:13,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233404 2024-11-24T20:47:13,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233404 2024-11-24T20:47:13,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233404) roll requested 2024-11-24T20:47:13,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233411 2024-11-24T20:47:13,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233411 2024-11-24T20:47:13,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233411) roll requested 2024-11-24T20:47:13,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233421 2024-11-24T20:47:13,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233421 2024-11-24T20:47:13,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233421) roll requested 2024-11-24T20:47:13,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233426 2024-11-24T20:47:13,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233426 2024-11-24T20:47:13,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233426) roll requested 2024-11-24T20:47:13,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233456 2024-11-24T20:47:13,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233456 2024-11-24T20:47:13,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233456) roll requested 2024-11-24T20:47:13,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233469 2024-11-24T20:47:13,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233469 2024-11-24T20:47:13,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233469) roll requested 2024-11-24T20:47:13,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233477 2024-11-24T20:47:13,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233477 2024-11-24T20:47:13,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233477) roll requested 2024-11-24T20:47:13,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233497 2024-11-24T20:47:13,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233497 2024-11-24T20:47:13,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233497) roll requested 2024-11-24T20:47:13,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233509 2024-11-24T20:47:13,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233509 2024-11-24T20:47:13,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233509) roll requested 2024-11-24T20:47:13,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233520 2024-11-24T20:47:13,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233520 2024-11-24T20:47:13,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233520) roll requested 2024-11-24T20:47:13,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233536 2024-11-24T20:47:13,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233536 2024-11-24T20:47:13,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233536) roll requested 2024-11-24T20:47:13,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233548 2024-11-24T20:47:13,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233548 2024-11-24T20:47:13,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233548) roll requested 2024-11-24T20:47:13,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233561 2024-11-24T20:47:13,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233561 2024-11-24T20:47:13,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233561) roll requested 2024-11-24T20:47:13,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233568 2024-11-24T20:47:13,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233568 2024-11-24T20:47:13,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233568) roll requested 2024-11-24T20:47:13,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233577 2024-11-24T20:47:13,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233577 2024-11-24T20:47:13,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233577) roll requested 2024-11-24T20:47:13,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233588 2024-11-24T20:47:13,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233588 2024-11-24T20:47:13,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233588) roll requested 2024-11-24T20:47:13,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233596 2024-11-24T20:47:13,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233596 2024-11-24T20:47:13,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233596) roll requested 2024-11-24T20:47:13,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233612 2024-11-24T20:47:13,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233612 2024-11-24T20:47:13,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233612) roll requested 2024-11-24T20:47:13,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233621 2024-11-24T20:47:13,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233621 2024-11-24T20:47:13,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233621) roll requested 2024-11-24T20:47:13,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233628 2024-11-24T20:47:13,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233628 2024-11-24T20:47:13,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233628) roll requested 2024-11-24T20:47:13,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233640 2024-11-24T20:47:13,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233640 2024-11-24T20:47:13,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233640) roll requested 2024-11-24T20:47:13,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233650 2024-11-24T20:47:13,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233650 2024-11-24T20:47:13,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233650) roll requested 2024-11-24T20:47:13,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233660 2024-11-24T20:47:13,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233660 2024-11-24T20:47:13,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233660) roll requested 2024-11-24T20:47:13,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233669 2024-11-24T20:47:13,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233669 2024-11-24T20:47:13,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233669) roll requested 2024-11-24T20:47:13,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233682 2024-11-24T20:47:13,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233682 2024-11-24T20:47:13,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233682) roll requested 2024-11-24T20:47:13,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233691 2024-11-24T20:47:13,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233691 2024-11-24T20:47:13,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233691) roll requested 2024-11-24T20:47:13,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233705 2024-11-24T20:47:13,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233705 2024-11-24T20:47:13,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233705) roll requested 2024-11-24T20:47:13,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233717 2024-11-24T20:47:13,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233717 2024-11-24T20:47:13,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233717) roll requested 2024-11-24T20:47:13,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233724 2024-11-24T20:47:13,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233724 2024-11-24T20:47:13,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233724) roll requested 2024-11-24T20:47:13,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233731 2024-11-24T20:47:13,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233731 2024-11-24T20:47:13,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233731) roll requested 2024-11-24T20:47:13,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233740 2024-11-24T20:47:13,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233740 2024-11-24T20:47:13,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233740) roll requested 2024-11-24T20:47:13,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233761 2024-11-24T20:47:13,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233761 2024-11-24T20:47:13,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233761) roll requested 2024-11-24T20:47:13,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233772 2024-11-24T20:47:13,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233772 2024-11-24T20:47:13,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233772) roll requested 2024-11-24T20:47:13,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233788 2024-11-24T20:47:13,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233788 2024-11-24T20:47:13,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233788) roll requested 2024-11-24T20:47:13,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233808 2024-11-24T20:47:13,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233808 2024-11-24T20:47:13,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233808) roll requested 2024-11-24T20:47:13,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233818 2024-11-24T20:47:13,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233818 2024-11-24T20:47:13,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233818) roll requested 2024-11-24T20:47:13,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233828 2024-11-24T20:47:13,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233828 2024-11-24T20:47:13,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233828) roll requested 2024-11-24T20:47:13,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233836 2024-11-24T20:47:13,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233836 2024-11-24T20:47:13,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233836) roll requested 2024-11-24T20:47:13,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233845 2024-11-24T20:47:13,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233845 2024-11-24T20:47:13,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233845) roll requested 2024-11-24T20:47:13,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233852 2024-11-24T20:47:13,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233852 2024-11-24T20:47:13,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233852) roll requested 2024-11-24T20:47:13,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233860 2024-11-24T20:47:13,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233860 2024-11-24T20:47:13,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233860) roll requested 2024-11-24T20:47:13,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233867 2024-11-24T20:47:13,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233867 2024-11-24T20:47:13,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233867) roll requested 2024-11-24T20:47:13,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233876 2024-11-24T20:47:13,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233876 2024-11-24T20:47:13,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233876) roll requested 2024-11-24T20:47:13,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233884 2024-11-24T20:47:13,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233884 2024-11-24T20:47:13,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233884) roll requested 2024-11-24T20:47:13,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233901 2024-11-24T20:47:13,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233901 2024-11-24T20:47:13,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233901) roll requested 2024-11-24T20:47:13,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233915 2024-11-24T20:47:13,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233915 2024-11-24T20:47:13,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233915) roll requested 2024-11-24T20:47:13,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233933 2024-11-24T20:47:13,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233933 2024-11-24T20:47:13,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233933) roll requested 2024-11-24T20:47:13,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233949 2024-11-24T20:47:13,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233949 2024-11-24T20:47:13,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233949) roll requested 2024-11-24T20:47:13,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233968 2024-11-24T20:47:13,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233968 2024-11-24T20:47:13,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233968) roll requested 2024-11-24T20:47:13,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233980 2024-11-24T20:47:13,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:13,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233980 2024-11-24T20:47:13,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:13,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:13,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233980) roll requested 2024-11-24T20:47:13,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481233992 2024-11-24T20:47:14,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233992 2024-11-24T20:47:14,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481233992) roll requested 2024-11-24T20:47:14,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234010 2024-11-24T20:47:14,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481233992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234010 2024-11-24T20:47:14,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234010) roll requested 2024-11-24T20:47:14,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234017 2024-11-24T20:47:14,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234017 2024-11-24T20:47:14,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234017) roll requested 2024-11-24T20:47:14,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234033 2024-11-24T20:47:14,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234033 2024-11-24T20:47:14,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234033) roll requested 2024-11-24T20:47:14,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234045 2024-11-24T20:47:14,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234045 2024-11-24T20:47:14,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234045) roll requested 2024-11-24T20:47:14,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234057 2024-11-24T20:47:14,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234057 2024-11-24T20:47:14,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234057) roll requested 2024-11-24T20:47:14,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234065 2024-11-24T20:47:14,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234065 2024-11-24T20:47:14,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234065) roll requested 2024-11-24T20:47:14,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234078 2024-11-24T20:47:14,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234078 2024-11-24T20:47:14,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234078) roll requested 2024-11-24T20:47:14,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234095 2024-11-24T20:47:14,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234095 2024-11-24T20:47:14,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234095) roll requested 2024-11-24T20:47:14,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234104 2024-11-24T20:47:14,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234104 2024-11-24T20:47:14,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234104) roll requested 2024-11-24T20:47:14,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234117 2024-11-24T20:47:14,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234117 2024-11-24T20:47:14,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234117) roll requested 2024-11-24T20:47:14,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234173 2024-11-24T20:47:14,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234173 2024-11-24T20:47:14,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234173) roll requested 2024-11-24T20:47:14,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234181 2024-11-24T20:47:14,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234181 2024-11-24T20:47:14,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234181) roll requested 2024-11-24T20:47:14,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234197 2024-11-24T20:47:14,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234197 2024-11-24T20:47:14,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234197) roll requested 2024-11-24T20:47:14,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234211 2024-11-24T20:47:14,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234211 2024-11-24T20:47:14,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234211) roll requested 2024-11-24T20:47:14,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234221 2024-11-24T20:47:14,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234221 2024-11-24T20:47:14,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234221) roll requested 2024-11-24T20:47:14,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234228 2024-11-24T20:47:14,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234228 2024-11-24T20:47:14,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234228) roll requested 2024-11-24T20:47:14,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234240 2024-11-24T20:47:14,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234240 2024-11-24T20:47:14,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234240) roll requested 2024-11-24T20:47:14,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234249 2024-11-24T20:47:14,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234249 2024-11-24T20:47:14,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234249) roll requested 2024-11-24T20:47:14,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234257 2024-11-24T20:47:14,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234257 2024-11-24T20:47:14,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234257) roll requested 2024-11-24T20:47:14,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234268 2024-11-24T20:47:14,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234268 2024-11-24T20:47:14,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234268) roll requested 2024-11-24T20:47:14,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234277 2024-11-24T20:47:14,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234277 2024-11-24T20:47:14,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234277) roll requested 2024-11-24T20:47:14,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234285 2024-11-24T20:47:14,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234285 2024-11-24T20:47:14,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234285) roll requested 2024-11-24T20:47:14,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234298 2024-11-24T20:47:14,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234298 2024-11-24T20:47:14,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234298) roll requested 2024-11-24T20:47:14,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234308 2024-11-24T20:47:14,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234308 2024-11-24T20:47:14,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234308) roll requested 2024-11-24T20:47:14,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234317 2024-11-24T20:47:14,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234317 2024-11-24T20:47:14,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234317) roll requested 2024-11-24T20:47:14,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234323 2024-11-24T20:47:14,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234323 2024-11-24T20:47:14,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234323) roll requested 2024-11-24T20:47:14,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234329 2024-11-24T20:47:14,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234329 2024-11-24T20:47:14,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234329) roll requested 2024-11-24T20:47:14,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234340 2024-11-24T20:47:14,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234340 2024-11-24T20:47:14,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234340) roll requested 2024-11-24T20:47:14,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234350 2024-11-24T20:47:14,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234350 2024-11-24T20:47:14,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234350) roll requested 2024-11-24T20:47:14,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234357 2024-11-24T20:47:14,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234357 2024-11-24T20:47:14,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234357) roll requested 2024-11-24T20:47:14,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234373 2024-11-24T20:47:14,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234373 2024-11-24T20:47:14,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234373) roll requested 2024-11-24T20:47:14,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234380 2024-11-24T20:47:14,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234380 2024-11-24T20:47:14,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234380) roll requested 2024-11-24T20:47:14,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234388 2024-11-24T20:47:14,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234388 2024-11-24T20:47:14,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234388) roll requested 2024-11-24T20:47:14,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234394 2024-11-24T20:47:14,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234394 2024-11-24T20:47:14,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234394) roll requested 2024-11-24T20:47:14,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234401 2024-11-24T20:47:14,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234401 2024-11-24T20:47:14,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234401) roll requested 2024-11-24T20:47:14,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234409 2024-11-24T20:47:14,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234409 2024-11-24T20:47:14,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234409) roll requested 2024-11-24T20:47:14,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234417 2024-11-24T20:47:14,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234417 2024-11-24T20:47:14,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234417) roll requested 2024-11-24T20:47:14,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234425 2024-11-24T20:47:14,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234425 2024-11-24T20:47:14,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234425) roll requested 2024-11-24T20:47:14,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234430 2024-11-24T20:47:14,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234430 2024-11-24T20:47:14,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234430) roll requested 2024-11-24T20:47:14,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234438 2024-11-24T20:47:14,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234438 2024-11-24T20:47:14,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234438) roll requested 2024-11-24T20:47:14,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234457 2024-11-24T20:47:14,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234457 2024-11-24T20:47:14,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234457) roll requested 2024-11-24T20:47:14,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234469 2024-11-24T20:47:14,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234469 2024-11-24T20:47:14,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234469) roll requested 2024-11-24T20:47:14,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234481 2024-11-24T20:47:14,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234481 2024-11-24T20:47:14,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234481) roll requested 2024-11-24T20:47:14,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234489 2024-11-24T20:47:14,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234489 2024-11-24T20:47:14,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234489) roll requested 2024-11-24T20:47:14,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234501 2024-11-24T20:47:14,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234501 2024-11-24T20:47:14,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234501) roll requested 2024-11-24T20:47:14,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234509 2024-11-24T20:47:14,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234509 2024-11-24T20:47:14,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234509) roll requested 2024-11-24T20:47:14,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234521 2024-11-24T20:47:14,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234521 2024-11-24T20:47:14,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234521) roll requested 2024-11-24T20:47:14,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234529 2024-11-24T20:47:14,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234529 2024-11-24T20:47:14,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234529) roll requested 2024-11-24T20:47:14,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234541 2024-11-24T20:47:14,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234541 2024-11-24T20:47:14,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234541) roll requested 2024-11-24T20:47:14,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234549 2024-11-24T20:47:14,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234549 2024-11-24T20:47:14,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234549) roll requested 2024-11-24T20:47:14,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234555 2024-11-24T20:47:14,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234555 2024-11-24T20:47:14,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234555) roll requested 2024-11-24T20:47:14,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234564 2024-11-24T20:47:14,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234564 2024-11-24T20:47:14,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234564) roll requested 2024-11-24T20:47:14,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234571 2024-11-24T20:47:14,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234571 2024-11-24T20:47:14,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234571) roll requested 2024-11-24T20:47:14,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234581 2024-11-24T20:47:14,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234581 2024-11-24T20:47:14,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234581) roll requested 2024-11-24T20:47:14,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234587 2024-11-24T20:47:14,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234587 2024-11-24T20:47:14,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234587) roll requested 2024-11-24T20:47:14,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234597 2024-11-24T20:47:14,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234597 2024-11-24T20:47:14,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234597) roll requested 2024-11-24T20:47:14,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234604 2024-11-24T20:47:14,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234604 2024-11-24T20:47:14,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234604) roll requested 2024-11-24T20:47:14,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234613 2024-11-24T20:47:14,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234613 2024-11-24T20:47:14,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234613) roll requested 2024-11-24T20:47:14,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234625 2024-11-24T20:47:14,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234625 2024-11-24T20:47:14,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234625) roll requested 2024-11-24T20:47:14,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234633 2024-11-24T20:47:14,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234633 2024-11-24T20:47:14,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234633) roll requested 2024-11-24T20:47:14,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234654 2024-11-24T20:47:14,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234654 2024-11-24T20:47:14,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234654) roll requested 2024-11-24T20:47:14,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234665 2024-11-24T20:47:14,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234665 2024-11-24T20:47:14,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234665) roll requested 2024-11-24T20:47:14,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234673 2024-11-24T20:47:14,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234673 2024-11-24T20:47:14,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234673) roll requested 2024-11-24T20:47:14,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234685 2024-11-24T20:47:14,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234685 2024-11-24T20:47:14,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234685) roll requested 2024-11-24T20:47:14,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234692 2024-11-24T20:47:14,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234692 2024-11-24T20:47:14,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234692) roll requested 2024-11-24T20:47:14,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234697 2024-11-24T20:47:14,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234697 2024-11-24T20:47:14,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234697) roll requested 2024-11-24T20:47:14,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234709 2024-11-24T20:47:14,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234709 2024-11-24T20:47:14,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234709) roll requested 2024-11-24T20:47:14,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234717 2024-11-24T20:47:14,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234717 2024-11-24T20:47:14,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234717) roll requested 2024-11-24T20:47:14,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234725 2024-11-24T20:47:14,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234725 2024-11-24T20:47:14,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234725) roll requested 2024-11-24T20:47:14,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234732 2024-11-24T20:47:14,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234732 2024-11-24T20:47:14,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234732) roll requested 2024-11-24T20:47:14,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234744 2024-11-24T20:47:14,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234744 2024-11-24T20:47:14,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234744) roll requested 2024-11-24T20:47:14,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234764 2024-11-24T20:47:14,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234764 2024-11-24T20:47:14,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234764) roll requested 2024-11-24T20:47:14,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234777 2024-11-24T20:47:14,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234777 2024-11-24T20:47:14,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234777) roll requested 2024-11-24T20:47:14,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234787 2024-11-24T20:47:14,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234787 2024-11-24T20:47:14,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234787) roll requested 2024-11-24T20:47:14,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234801 2024-11-24T20:47:14,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234801 2024-11-24T20:47:14,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234801) roll requested 2024-11-24T20:47:14,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234810 2024-11-24T20:47:14,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234810 2024-11-24T20:47:14,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234810) roll requested 2024-11-24T20:47:14,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234819 2024-11-24T20:47:14,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234819 2024-11-24T20:47:14,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234819) roll requested 2024-11-24T20:47:14,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234829 2024-11-24T20:47:14,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234829 2024-11-24T20:47:14,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234829) roll requested 2024-11-24T20:47:14,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234842 2024-11-24T20:47:14,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234842 2024-11-24T20:47:14,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234842) roll requested 2024-11-24T20:47:14,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234849 2024-11-24T20:47:14,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234849 2024-11-24T20:47:14,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234849) roll requested 2024-11-24T20:47:14,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234861 2024-11-24T20:47:14,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234861 2024-11-24T20:47:14,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234861) roll requested 2024-11-24T20:47:14,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234873 2024-11-24T20:47:14,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234873 2024-11-24T20:47:14,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234873) roll requested 2024-11-24T20:47:14,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234881 2024-11-24T20:47:14,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234881 2024-11-24T20:47:14,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234881) roll requested 2024-11-24T20:47:14,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234894 2024-11-24T20:47:14,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234894 2024-11-24T20:47:14,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234894) roll requested 2024-11-24T20:47:14,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234900 2024-11-24T20:47:14,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234900 2024-11-24T20:47:14,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234900) roll requested 2024-11-24T20:47:14,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234909 2024-11-24T20:47:14,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234909 2024-11-24T20:47:14,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234909) roll requested 2024-11-24T20:47:14,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234920 2024-11-24T20:47:14,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234920 2024-11-24T20:47:14,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234920) roll requested 2024-11-24T20:47:14,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234927 2024-11-24T20:47:14,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234927 2024-11-24T20:47:14,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234927) roll requested 2024-11-24T20:47:14,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234949 2024-11-24T20:47:14,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234949 2024-11-24T20:47:14,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234949) roll requested 2024-11-24T20:47:14,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234961 2024-11-24T20:47:14,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234961 2024-11-24T20:47:14,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234961) roll requested 2024-11-24T20:47:14,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234970 2024-11-24T20:47:14,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234970 2024-11-24T20:47:14,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234970) roll requested 2024-11-24T20:47:14,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234988 2024-11-24T20:47:14,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:14,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234988 2024-11-24T20:47:14,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:14,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:14,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:14,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:14,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234988) roll requested 2024-11-24T20:47:14,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481234998 2024-11-24T20:47:15,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234998 2024-11-24T20:47:15,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481234998) roll requested 2024-11-24T20:47:15,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235017 2024-11-24T20:47:15,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481234998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235017 2024-11-24T20:47:15,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235017) roll requested 2024-11-24T20:47:15,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235033 2024-11-24T20:47:15,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235033 2024-11-24T20:47:15,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235033) roll requested 2024-11-24T20:47:15,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235056 2024-11-24T20:47:15,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235056 2024-11-24T20:47:15,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235056) roll requested 2024-11-24T20:47:15,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235069 2024-11-24T20:47:15,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235069 2024-11-24T20:47:15,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235069) roll requested 2024-11-24T20:47:15,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235082 2024-11-24T20:47:15,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235082 2024-11-24T20:47:15,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235082) roll requested 2024-11-24T20:47:15,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235093 2024-11-24T20:47:15,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235093 2024-11-24T20:47:15,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235093) roll requested 2024-11-24T20:47:15,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235116 2024-11-24T20:47:15,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235116 2024-11-24T20:47:15,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235116) roll requested 2024-11-24T20:47:15,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235133 2024-11-24T20:47:15,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235133 2024-11-24T20:47:15,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235133) roll requested 2024-11-24T20:47:15,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235170 2024-11-24T20:47:15,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235170 2024-11-24T20:47:15,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235170) roll requested 2024-11-24T20:47:15,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235190 2024-11-24T20:47:15,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235190 2024-11-24T20:47:15,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235190) roll requested 2024-11-24T20:47:15,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235205 2024-11-24T20:47:15,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235205 2024-11-24T20:47:15,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235205) roll requested 2024-11-24T20:47:15,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235221 2024-11-24T20:47:15,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235221 2024-11-24T20:47:15,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235221) roll requested 2024-11-24T20:47:15,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235233 2024-11-24T20:47:15,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235233 2024-11-24T20:47:15,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235233) roll requested 2024-11-24T20:47:15,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235245 2024-11-24T20:47:15,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235245 2024-11-24T20:47:15,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235245) roll requested 2024-11-24T20:47:15,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235253 2024-11-24T20:47:15,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235253 2024-11-24T20:47:15,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235253) roll requested 2024-11-24T20:47:15,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235261 2024-11-24T20:47:15,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235261 2024-11-24T20:47:15,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235261) roll requested 2024-11-24T20:47:15,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235269 2024-11-24T20:47:15,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235269 2024-11-24T20:47:15,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235269) roll requested 2024-11-24T20:47:15,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235277 2024-11-24T20:47:15,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235277 2024-11-24T20:47:15,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235277) roll requested 2024-11-24T20:47:15,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235285 2024-11-24T20:47:15,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235285 2024-11-24T20:47:15,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235285) roll requested 2024-11-24T20:47:15,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235291 2024-11-24T20:47:15,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235291 2024-11-24T20:47:15,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235291) roll requested 2024-11-24T20:47:15,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235297 2024-11-24T20:47:15,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235297 2024-11-24T20:47:15,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235297) roll requested 2024-11-24T20:47:15,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235304 2024-11-24T20:47:15,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235304 2024-11-24T20:47:15,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235304) roll requested 2024-11-24T20:47:15,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235313 2024-11-24T20:47:15,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235313 2024-11-24T20:47:15,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235313) roll requested 2024-11-24T20:47:15,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235320 2024-11-24T20:47:15,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235320 2024-11-24T20:47:15,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235320) roll requested 2024-11-24T20:47:15,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235329 2024-11-24T20:47:15,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235329 2024-11-24T20:47:15,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235329) roll requested 2024-11-24T20:47:15,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235336 2024-11-24T20:47:15,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235336 2024-11-24T20:47:15,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235336) roll requested 2024-11-24T20:47:15,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235342 2024-11-24T20:47:15,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235342 2024-11-24T20:47:15,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235342) roll requested 2024-11-24T20:47:15,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235349 2024-11-24T20:47:15,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235349 2024-11-24T20:47:15,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235349) roll requested 2024-11-24T20:47:15,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235354 2024-11-24T20:47:15,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235354 2024-11-24T20:47:15,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235354) roll requested 2024-11-24T20:47:15,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235361 2024-11-24T20:47:15,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235361 2024-11-24T20:47:15,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235361) roll requested 2024-11-24T20:47:15,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235367 2024-11-24T20:47:15,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235367 2024-11-24T20:47:15,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235367) roll requested 2024-11-24T20:47:15,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235377 2024-11-24T20:47:15,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235377 2024-11-24T20:47:15,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235377) roll requested 2024-11-24T20:47:15,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235384 2024-11-24T20:47:15,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235384 2024-11-24T20:47:15,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235384) roll requested 2024-11-24T20:47:15,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235393 2024-11-24T20:47:15,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235393 2024-11-24T20:47:15,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235393) roll requested 2024-11-24T20:47:15,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235401 2024-11-24T20:47:15,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235401 2024-11-24T20:47:15,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235401) roll requested 2024-11-24T20:47:15,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235409 2024-11-24T20:47:15,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235409 2024-11-24T20:47:15,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235409) roll requested 2024-11-24T20:47:15,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235427 2024-11-24T20:47:15,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235427 2024-11-24T20:47:15,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235427) roll requested 2024-11-24T20:47:15,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235482 2024-11-24T20:47:15,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235482 2024-11-24T20:47:15,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235482) roll requested 2024-11-24T20:47:15,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235521 2024-11-24T20:47:15,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235521 2024-11-24T20:47:15,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235521) roll requested 2024-11-24T20:47:15,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235540 2024-11-24T20:47:15,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235540 2024-11-24T20:47:15,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235540) roll requested 2024-11-24T20:47:15,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235553 2024-11-24T20:47:15,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235553 2024-11-24T20:47:15,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235553) roll requested 2024-11-24T20:47:15,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235595 2024-11-24T20:47:15,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235595 2024-11-24T20:47:15,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,613 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235595) roll requested 2024-11-24T20:47:15,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235614 2024-11-24T20:47:15,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235614 2024-11-24T20:47:15,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235614) roll requested 2024-11-24T20:47:15,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235628 2024-11-24T20:47:15,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235628 2024-11-24T20:47:15,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235628) roll requested 2024-11-24T20:47:15,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235634 2024-11-24T20:47:15,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235634 2024-11-24T20:47:15,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235634) roll requested 2024-11-24T20:47:15,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235649 2024-11-24T20:47:15,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235649 2024-11-24T20:47:15,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235649) roll requested 2024-11-24T20:47:15,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235662 2024-11-24T20:47:15,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235662 2024-11-24T20:47:15,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235662) roll requested 2024-11-24T20:47:15,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235674 2024-11-24T20:47:15,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235674 2024-11-24T20:47:15,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235674) roll requested 2024-11-24T20:47:15,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235689 2024-11-24T20:47:15,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235689 2024-11-24T20:47:15,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235689) roll requested 2024-11-24T20:47:15,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235704 2024-11-24T20:47:15,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235704 2024-11-24T20:47:15,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,711 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,711 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235704) roll requested 2024-11-24T20:47:15,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235711 2024-11-24T20:47:15,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235711 2024-11-24T20:47:15,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235711) roll requested 2024-11-24T20:47:15,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235720 2024-11-24T20:47:15,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235711 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235720 2024-11-24T20:47:15,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235720) roll requested 2024-11-24T20:47:15,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235731 2024-11-24T20:47:15,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235731 2024-11-24T20:47:15,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235731) roll requested 2024-11-24T20:47:15,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235744 2024-11-24T20:47:15,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235744 2024-11-24T20:47:15,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235744) roll requested 2024-11-24T20:47:15,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235753 2024-11-24T20:47:15,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235753 2024-11-24T20:47:15,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235753) roll requested 2024-11-24T20:47:15,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235787 2024-11-24T20:47:15,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235787 2024-11-24T20:47:15,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235787) roll requested 2024-11-24T20:47:15,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235807 2024-11-24T20:47:15,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235807 2024-11-24T20:47:15,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235807) roll requested 2024-11-24T20:47:15,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235830 2024-11-24T20:47:15,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235830 2024-11-24T20:47:15,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235830) roll requested 2024-11-24T20:47:15,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235849 2024-11-24T20:47:15,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235849 2024-11-24T20:47:15,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235849) roll requested 2024-11-24T20:47:15,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235905 2024-11-24T20:47:15,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235905 2024-11-24T20:47:15,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235905) roll requested 2024-11-24T20:47:15,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235912 2024-11-24T20:47:15,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235912 2024-11-24T20:47:15,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235912) roll requested 2024-11-24T20:47:15,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235924 2024-11-24T20:47:15,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235924 2024-11-24T20:47:15,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235924) roll requested 2024-11-24T20:47:15,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235953 2024-11-24T20:47:15,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235953 2024-11-24T20:47:15,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235953) roll requested 2024-11-24T20:47:15,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235972 2024-11-24T20:47:15,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235972 2024-11-24T20:47:15,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235972) roll requested 2024-11-24T20:47:15,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235985 2024-11-24T20:47:15,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:15,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235985 2024-11-24T20:47:15,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:15,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:15,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:15,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:15,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235985) roll requested 2024-11-24T20:47:15,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481235997 2024-11-24T20:47:16,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235997 2024-11-24T20:47:16,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481235997) roll requested 2024-11-24T20:47:16,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236009 2024-11-24T20:47:16,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481235997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236009 2024-11-24T20:47:16,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236009) roll requested 2024-11-24T20:47:16,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236018 2024-11-24T20:47:16,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236018 2024-11-24T20:47:16,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236018) roll requested 2024-11-24T20:47:16,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236029 2024-11-24T20:47:16,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236029 2024-11-24T20:47:16,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236029) roll requested 2024-11-24T20:47:16,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236037 2024-11-24T20:47:16,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236037 2024-11-24T20:47:16,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236037) roll requested 2024-11-24T20:47:16,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236045 2024-11-24T20:47:16,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236045 2024-11-24T20:47:16,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236045) roll requested 2024-11-24T20:47:16,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236053 2024-11-24T20:47:16,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236053 2024-11-24T20:47:16,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236053) roll requested 2024-11-24T20:47:16,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236061 2024-11-24T20:47:16,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236061 2024-11-24T20:47:16,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236061) roll requested 2024-11-24T20:47:16,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236068 2024-11-24T20:47:16,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236068 2024-11-24T20:47:16,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236068) roll requested 2024-11-24T20:47:16,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236081 2024-11-24T20:47:16,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236081 2024-11-24T20:47:16,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,088 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236081) roll requested 2024-11-24T20:47:16,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236088 2024-11-24T20:47:16,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236088 2024-11-24T20:47:16,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236088) roll requested 2024-11-24T20:47:16,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236097 2024-11-24T20:47:16,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236097 2024-11-24T20:47:16,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236097) roll requested 2024-11-24T20:47:16,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236104 2024-11-24T20:47:16,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236104 2024-11-24T20:47:16,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236104) roll requested 2024-11-24T20:47:16,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236111 2024-11-24T20:47:16,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236111 2024-11-24T20:47:16,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236111) roll requested 2024-11-24T20:47:16,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236117 2024-11-24T20:47:16,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236117 2024-11-24T20:47:16,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236117) roll requested 2024-11-24T20:47:16,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236124 2024-11-24T20:47:16,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236124 2024-11-24T20:47:16,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236124) roll requested 2024-11-24T20:47:16,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236130 2024-11-24T20:47:16,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236130 2024-11-24T20:47:16,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236124 is not closed yet, will try archiving it next time 2024-11-24T20:47:16,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236130) roll requested 2024-11-24T20:47:16,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236136 2024-11-24T20:47:16,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236136 2024-11-24T20:47:16,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236136) roll requested 2024-11-24T20:47:16,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236143 2024-11-24T20:47:16,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236143 2024-11-24T20:47:16,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236143) roll requested 2024-11-24T20:47:16,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236149 2024-11-24T20:47:16,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236149 2024-11-24T20:47:16,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236149) roll requested 2024-11-24T20:47:16,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236165 2024-11-24T20:47:16,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236165 2024-11-24T20:47:16,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236165) roll requested 2024-11-24T20:47:16,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236172 2024-11-24T20:47:16,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236172 2024-11-24T20:47:16,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236172) roll requested 2024-11-24T20:47:16,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236179 2024-11-24T20:47:16,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236179 2024-11-24T20:47:16,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236179) roll requested 2024-11-24T20:47:16,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236192 2024-11-24T20:47:16,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236192 2024-11-24T20:47:16,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236192) roll requested 2024-11-24T20:47:16,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236199 2024-11-24T20:47:16,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236199 2024-11-24T20:47:16,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236199) roll requested 2024-11-24T20:47:16,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236209 2024-11-24T20:47:16,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236209 2024-11-24T20:47:16,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236209) roll requested 2024-11-24T20:47:16,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236217 2024-11-24T20:47:16,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236217 2024-11-24T20:47:16,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236217) roll requested 2024-11-24T20:47:16,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236223 2024-11-24T20:47:16,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236223 2024-11-24T20:47:16,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236223) roll requested 2024-11-24T20:47:16,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236233 2024-11-24T20:47:16,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236233 2024-11-24T20:47:16,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236233) roll requested 2024-11-24T20:47:16,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236243 2024-11-24T20:47:16,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236243 2024-11-24T20:47:16,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236243) roll requested 2024-11-24T20:47:16,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236252 2024-11-24T20:47:16,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236252 2024-11-24T20:47:16,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236252) roll requested 2024-11-24T20:47:16,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236272 2024-11-24T20:47:16,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236272 2024-11-24T20:47:16,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236272) roll requested 2024-11-24T20:47:16,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236284 2024-11-24T20:47:16,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236284 2024-11-24T20:47:16,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236284) roll requested 2024-11-24T20:47:16,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236291 2024-11-24T20:47:16,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236291 2024-11-24T20:47:16,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236291) roll requested 2024-11-24T20:47:16,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236299 2024-11-24T20:47:16,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236299 2024-11-24T20:47:16,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236299) roll requested 2024-11-24T20:47:16,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236309 2024-11-24T20:47:16,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236309 2024-11-24T20:47:16,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236309) roll requested 2024-11-24T20:47:16,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236317 2024-11-24T20:47:16,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236317 2024-11-24T20:47:16,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236317) roll requested 2024-11-24T20:47:16,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236324 2024-11-24T20:47:16,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236324 2024-11-24T20:47:16,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236324) roll requested 2024-11-24T20:47:16,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236341 2024-11-24T20:47:16,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236341 2024-11-24T20:47:16,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236341) roll requested 2024-11-24T20:47:16,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236356 2024-11-24T20:47:16,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236356 2024-11-24T20:47:16,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236356) roll requested 2024-11-24T20:47:16,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236389 2024-11-24T20:47:16,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236389 2024-11-24T20:47:16,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236389) roll requested 2024-11-24T20:47:16,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236404 2024-11-24T20:47:16,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236404 2024-11-24T20:47:16,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236389 is not closed yet, will try archiving it next time 2024-11-24T20:47:16,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236404) roll requested 2024-11-24T20:47:16,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236425 2024-11-24T20:47:16,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236425 2024-11-24T20:47:16,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236425) roll requested 2024-11-24T20:47:16,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236437 2024-11-24T20:47:16,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236437 2024-11-24T20:47:16,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236437) roll requested 2024-11-24T20:47:16,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236454 2024-11-24T20:47:16,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236454 2024-11-24T20:47:16,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236454) roll requested 2024-11-24T20:47:16,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236461 2024-11-24T20:47:16,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236461 2024-11-24T20:47:16,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236461) roll requested 2024-11-24T20:47:16,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236474 2024-11-24T20:47:16,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236474 2024-11-24T20:47:16,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236474) roll requested 2024-11-24T20:47:16,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236484 2024-11-24T20:47:16,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236484 2024-11-24T20:47:16,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236484) roll requested 2024-11-24T20:47:16,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236500 2024-11-24T20:47:16,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236500 2024-11-24T20:47:16,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236500) roll requested 2024-11-24T20:47:16,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236509 2024-11-24T20:47:16,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236509 2024-11-24T20:47:16,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236509) roll requested 2024-11-24T20:47:16,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236520 2024-11-24T20:47:16,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236520 2024-11-24T20:47:16,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236520) roll requested 2024-11-24T20:47:16,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236527 2024-11-24T20:47:16,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236527 2024-11-24T20:47:16,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236527) roll requested 2024-11-24T20:47:16,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236537 2024-11-24T20:47:16,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236537 2024-11-24T20:47:16,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,545 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236537) roll requested 2024-11-24T20:47:16,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236545 2024-11-24T20:47:16,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236545 2024-11-24T20:47:16,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236545) roll requested 2024-11-24T20:47:16,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236558 2024-11-24T20:47:16,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236558 2024-11-24T20:47:16,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236558) roll requested 2024-11-24T20:47:16,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236569 2024-11-24T20:47:16,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236569 2024-11-24T20:47:16,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236569) roll requested 2024-11-24T20:47:16,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236577 2024-11-24T20:47:16,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236577 2024-11-24T20:47:16,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236577) roll requested 2024-11-24T20:47:16,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236593 2024-11-24T20:47:16,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236593 2024-11-24T20:47:16,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236593) roll requested 2024-11-24T20:47:16,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236605 2024-11-24T20:47:16,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236605 2024-11-24T20:47:16,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236605) roll requested 2024-11-24T20:47:16,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236610 2024-11-24T20:47:16,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236610 2024-11-24T20:47:16,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236610) roll requested 2024-11-24T20:47:16,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236618 2024-11-24T20:47:16,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236618 2024-11-24T20:47:16,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236618) roll requested 2024-11-24T20:47:16,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236626 2024-11-24T20:47:16,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236626 2024-11-24T20:47:16,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236626) roll requested 2024-11-24T20:47:16,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236635 2024-11-24T20:47:16,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236635 2024-11-24T20:47:16,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236635) roll requested 2024-11-24T20:47:16,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236642 2024-11-24T20:47:16,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236642 2024-11-24T20:47:16,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236642) roll requested 2024-11-24T20:47:16,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236648 2024-11-24T20:47:16,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236648 2024-11-24T20:47:16,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236648) roll requested 2024-11-24T20:47:16,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236656 2024-11-24T20:47:16,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236656 2024-11-24T20:47:16,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236656) roll requested 2024-11-24T20:47:16,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236662 2024-11-24T20:47:16,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236662 2024-11-24T20:47:16,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236662) roll requested 2024-11-24T20:47:16,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236677 2024-11-24T20:47:16,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236677 2024-11-24T20:47:16,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236677) roll requested 2024-11-24T20:47:16,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236697 2024-11-24T20:47:16,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236697 2024-11-24T20:47:16,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236697) roll requested 2024-11-24T20:47:16,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236709 2024-11-24T20:47:16,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236709 2024-11-24T20:47:16,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236709) roll requested 2024-11-24T20:47:16,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236737 2024-11-24T20:47:16,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236737 2024-11-24T20:47:16,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236737) roll requested 2024-11-24T20:47:16,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236765 2024-11-24T20:47:16,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236765 2024-11-24T20:47:16,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236765) roll requested 2024-11-24T20:47:16,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236779 2024-11-24T20:47:16,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236779 2024-11-24T20:47:16,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236779) roll requested 2024-11-24T20:47:16,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236789 2024-11-24T20:47:16,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236789 2024-11-24T20:47:16,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236789) roll requested 2024-11-24T20:47:16,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236801 2024-11-24T20:47:16,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236801 2024-11-24T20:47:16,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236801) roll requested 2024-11-24T20:47:16,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236809 2024-11-24T20:47:16,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236809 2024-11-24T20:47:16,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236809) roll requested 2024-11-24T20:47:16,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236817 2024-11-24T20:47:16,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236817 2024-11-24T20:47:16,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236817) roll requested 2024-11-24T20:47:16,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236825 2024-11-24T20:47:16,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236825 2024-11-24T20:47:16,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236825) roll requested 2024-11-24T20:47:16,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236831 2024-11-24T20:47:16,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236831 2024-11-24T20:47:16,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236831) roll requested 2024-11-24T20:47:16,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236849 2024-11-24T20:47:16,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236849 2024-11-24T20:47:16,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236849) roll requested 2024-11-24T20:47:16,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236877 2024-11-24T20:47:16,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236877 2024-11-24T20:47:16,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236877) roll requested 2024-11-24T20:47:16,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236893 2024-11-24T20:47:16,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236893 2024-11-24T20:47:16,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236893) roll requested 2024-11-24T20:47:16,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236900 2024-11-24T20:47:16,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236900 2024-11-24T20:47:16,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236900) roll requested 2024-11-24T20:47:16,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236906 2024-11-24T20:47:16,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236906 2024-11-24T20:47:16,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236906) roll requested 2024-11-24T20:47:16,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236912 2024-11-24T20:47:16,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236912 2024-11-24T20:47:16,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236912) roll requested 2024-11-24T20:47:16,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236919 2024-11-24T20:47:16,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236919 2024-11-24T20:47:16,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236919) roll requested 2024-11-24T20:47:16,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236924 2024-11-24T20:47:16,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236924 2024-11-24T20:47:16,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236924) roll requested 2024-11-24T20:47:16,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236933 2024-11-24T20:47:16,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236933 2024-11-24T20:47:16,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236933) roll requested 2024-11-24T20:47:16,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236941 2024-11-24T20:47:16,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236941 2024-11-24T20:47:16,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236941) roll requested 2024-11-24T20:47:16,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236949 2024-11-24T20:47:16,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236949 2024-11-24T20:47:16,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236949) roll requested 2024-11-24T20:47:16,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236968 2024-11-24T20:47:16,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236968 2024-11-24T20:47:16,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:16,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:16,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:16,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:16,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236968) roll requested 2024-11-24T20:47:16,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481236989 2024-11-24T20:47:16,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:16,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236989 2024-11-24T20:47:17,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481236989) roll requested 2024-11-24T20:47:17,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237001 2024-11-24T20:47:17,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481236989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237001 2024-11-24T20:47:17,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237001) roll requested 2024-11-24T20:47:17,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237012 2024-11-24T20:47:17,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237012 2024-11-24T20:47:17,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237012) roll requested 2024-11-24T20:47:17,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237024 2024-11-24T20:47:17,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237024 2024-11-24T20:47:17,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237024) roll requested 2024-11-24T20:47:17,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237032 2024-11-24T20:47:17,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237032 2024-11-24T20:47:17,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237032) roll requested 2024-11-24T20:47:17,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237044 2024-11-24T20:47:17,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237044 2024-11-24T20:47:17,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237044) roll requested 2024-11-24T20:47:17,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237060 2024-11-24T20:47:17,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237060 2024-11-24T20:47:17,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237060) roll requested 2024-11-24T20:47:17,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237082 2024-11-24T20:47:17,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237082 2024-11-24T20:47:17,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237082) roll requested 2024-11-24T20:47:17,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237097 2024-11-24T20:47:17,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237097 2024-11-24T20:47:17,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237097) roll requested 2024-11-24T20:47:17,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237118 2024-11-24T20:47:17,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237118 2024-11-24T20:47:17,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237118) roll requested 2024-11-24T20:47:17,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237141 2024-11-24T20:47:17,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237141 2024-11-24T20:47:17,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237141) roll requested 2024-11-24T20:47:17,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237163 2024-11-24T20:47:17,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237163 2024-11-24T20:47:17,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237163) roll requested 2024-11-24T20:47:17,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237180 2024-11-24T20:47:17,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237180 2024-11-24T20:47:17,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237180) roll requested 2024-11-24T20:47:17,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237189 2024-11-24T20:47:17,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237189 2024-11-24T20:47:17,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237189) roll requested 2024-11-24T20:47:17,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237198 2024-11-24T20:47:17,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237198 2024-11-24T20:47:17,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237198) roll requested 2024-11-24T20:47:17,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237204 2024-11-24T20:47:17,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237204 2024-11-24T20:47:17,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237204) roll requested 2024-11-24T20:47:17,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237212 2024-11-24T20:47:17,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237212 2024-11-24T20:47:17,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237212) roll requested 2024-11-24T20:47:17,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237221 2024-11-24T20:47:17,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237221 2024-11-24T20:47:17,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237221) roll requested 2024-11-24T20:47:17,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237227 2024-11-24T20:47:17,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237227 2024-11-24T20:47:17,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237227) roll requested 2024-11-24T20:47:17,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237234 2024-11-24T20:47:17,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237234 2024-11-24T20:47:17,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237234) roll requested 2024-11-24T20:47:17,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237240 2024-11-24T20:47:17,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237240 2024-11-24T20:47:17,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237240) roll requested 2024-11-24T20:47:17,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237247 2024-11-24T20:47:17,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237247 2024-11-24T20:47:17,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237247) roll requested 2024-11-24T20:47:17,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237253 2024-11-24T20:47:17,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237253 2024-11-24T20:47:17,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237253) roll requested 2024-11-24T20:47:17,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237261 2024-11-24T20:47:17,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237261 2024-11-24T20:47:17,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237261) roll requested 2024-11-24T20:47:17,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237269 2024-11-24T20:47:17,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237269 2024-11-24T20:47:17,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237269) roll requested 2024-11-24T20:47:17,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237277 2024-11-24T20:47:17,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237277 2024-11-24T20:47:17,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237277) roll requested 2024-11-24T20:47:17,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237283 2024-11-24T20:47:17,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237283 2024-11-24T20:47:17,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237283) roll requested 2024-11-24T20:47:17,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237290 2024-11-24T20:47:17,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237290 2024-11-24T20:47:17,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237290) roll requested 2024-11-24T20:47:17,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237309 2024-11-24T20:47:17,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237309 2024-11-24T20:47:17,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237309) roll requested 2024-11-24T20:47:17,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237316 2024-11-24T20:47:17,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237316 2024-11-24T20:47:17,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237316) roll requested 2024-11-24T20:47:17,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237327 2024-11-24T20:47:17,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237327 2024-11-24T20:47:17,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237327) roll requested 2024-11-24T20:47:17,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237335 2024-11-24T20:47:17,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237335 2024-11-24T20:47:17,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237335) roll requested 2024-11-24T20:47:17,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237341 2024-11-24T20:47:17,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237341 2024-11-24T20:47:17,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237341) roll requested 2024-11-24T20:47:17,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237348 2024-11-24T20:47:17,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237348 2024-11-24T20:47:17,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237348) roll requested 2024-11-24T20:47:17,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237354 2024-11-24T20:47:17,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237354 2024-11-24T20:47:17,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237354) roll requested 2024-11-24T20:47:17,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237361 2024-11-24T20:47:17,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237361 2024-11-24T20:47:17,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237361) roll requested 2024-11-24T20:47:17,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237371 2024-11-24T20:47:17,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237371 2024-11-24T20:47:17,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237371) roll requested 2024-11-24T20:47:17,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237381 2024-11-24T20:47:17,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237381 2024-11-24T20:47:17,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237381) roll requested 2024-11-24T20:47:17,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237389 2024-11-24T20:47:17,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237389 2024-11-24T20:47:17,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237389) roll requested 2024-11-24T20:47:17,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237395 2024-11-24T20:47:17,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237395 2024-11-24T20:47:17,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237395) roll requested 2024-11-24T20:47:17,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237402 2024-11-24T20:47:17,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237402 2024-11-24T20:47:17,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237402) roll requested 2024-11-24T20:47:17,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237408 2024-11-24T20:47:17,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237408 2024-11-24T20:47:17,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237408) roll requested 2024-11-24T20:47:17,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237415 2024-11-24T20:47:17,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237415 2024-11-24T20:47:17,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237415) roll requested 2024-11-24T20:47:17,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237424 2024-11-24T20:47:17,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237424 2024-11-24T20:47:17,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237424) roll requested 2024-11-24T20:47:17,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237430 2024-11-24T20:47:17,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237430 2024-11-24T20:47:17,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237430) roll requested 2024-11-24T20:47:17,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237440 2024-11-24T20:47:17,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237440 2024-11-24T20:47:17,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237440) roll requested 2024-11-24T20:47:17,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237448 2024-11-24T20:47:17,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237448 2024-11-24T20:47:17,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237448) roll requested 2024-11-24T20:47:17,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237458 2024-11-24T20:47:17,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237458 2024-11-24T20:47:17,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237458) roll requested 2024-11-24T20:47:17,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237464 2024-11-24T20:47:17,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237464 2024-11-24T20:47:17,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237464) roll requested 2024-11-24T20:47:17,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237473 2024-11-24T20:47:17,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237473 2024-11-24T20:47:17,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237473) roll requested 2024-11-24T20:47:17,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237479 2024-11-24T20:47:17,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237479 2024-11-24T20:47:17,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237479) roll requested 2024-11-24T20:47:17,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237486 2024-11-24T20:47:17,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237486 2024-11-24T20:47:17,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237486) roll requested 2024-11-24T20:47:17,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237496 2024-11-24T20:47:17,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237496 2024-11-24T20:47:17,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237496) roll requested 2024-11-24T20:47:17,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237503 2024-11-24T20:47:17,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237503 2024-11-24T20:47:17,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237503) roll requested 2024-11-24T20:47:17,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237510 2024-11-24T20:47:17,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237510 2024-11-24T20:47:17,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237510) roll requested 2024-11-24T20:47:17,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237520 2024-11-24T20:47:17,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237520 2024-11-24T20:47:17,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237520) roll requested 2024-11-24T20:47:17,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237527 2024-11-24T20:47:17,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237527 2024-11-24T20:47:17,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237527) roll requested 2024-11-24T20:47:17,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237532 2024-11-24T20:47:17,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237532 2024-11-24T20:47:17,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237532) roll requested 2024-11-24T20:47:17,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237543 2024-11-24T20:47:17,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237543 2024-11-24T20:47:17,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237543) roll requested 2024-11-24T20:47:17,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237552 2024-11-24T20:47:17,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237552 2024-11-24T20:47:17,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237552) roll requested 2024-11-24T20:47:17,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237562 2024-11-24T20:47:17,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237562 2024-11-24T20:47:17,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237562) roll requested 2024-11-24T20:47:17,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237570 2024-11-24T20:47:17,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237570 2024-11-24T20:47:17,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237570) roll requested 2024-11-24T20:47:17,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237587 2024-11-24T20:47:17,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237587 2024-11-24T20:47:17,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237587) roll requested 2024-11-24T20:47:17,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237612 2024-11-24T20:47:17,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237612 2024-11-24T20:47:17,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237612) roll requested 2024-11-24T20:47:17,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237628 2024-11-24T20:47:17,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237628 2024-11-24T20:47:17,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237628) roll requested 2024-11-24T20:47:17,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237637 2024-11-24T20:47:17,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237637 2024-11-24T20:47:17,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237637) roll requested 2024-11-24T20:47:17,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237645 2024-11-24T20:47:17,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237645 2024-11-24T20:47:17,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237645) roll requested 2024-11-24T20:47:17,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237653 2024-11-24T20:47:17,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237653 2024-11-24T20:47:17,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237653) roll requested 2024-11-24T20:47:17,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237661 2024-11-24T20:47:17,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237661 2024-11-24T20:47:17,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237661) roll requested 2024-11-24T20:47:17,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237669 2024-11-24T20:47:17,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237669 2024-11-24T20:47:17,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237669) roll requested 2024-11-24T20:47:17,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237681 2024-11-24T20:47:17,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237681 2024-11-24T20:47:17,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237681) roll requested 2024-11-24T20:47:17,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237689 2024-11-24T20:47:17,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237689 2024-11-24T20:47:17,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237689) roll requested 2024-11-24T20:47:17,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237696 2024-11-24T20:47:17,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237696 2024-11-24T20:47:17,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237696) roll requested 2024-11-24T20:47:17,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237703 2024-11-24T20:47:17,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237703 2024-11-24T20:47:17,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237703) roll requested 2024-11-24T20:47:17,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237715 2024-11-24T20:47:17,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237715 2024-11-24T20:47:17,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237715) roll requested 2024-11-24T20:47:17,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237735 2024-11-24T20:47:17,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237735 2024-11-24T20:47:17,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237735) roll requested 2024-11-24T20:47:17,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237744 2024-11-24T20:47:17,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237744 2024-11-24T20:47:17,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237744) roll requested 2024-11-24T20:47:17,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237753 2024-11-24T20:47:17,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237753 2024-11-24T20:47:17,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237753) roll requested 2024-11-24T20:47:17,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237759 2024-11-24T20:47:17,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237759 2024-11-24T20:47:17,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237759) roll requested 2024-11-24T20:47:17,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237767 2024-11-24T20:47:17,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237767 2024-11-24T20:47:17,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237767) roll requested 2024-11-24T20:47:17,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237776 2024-11-24T20:47:17,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237776 2024-11-24T20:47:17,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237776) roll requested 2024-11-24T20:47:17,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237801 2024-11-24T20:47:17,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237801 2024-11-24T20:47:17,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237801) roll requested 2024-11-24T20:47:17,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237808 2024-11-24T20:47:17,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237808 2024-11-24T20:47:17,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237808) roll requested 2024-11-24T20:47:17,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237816 2024-11-24T20:47:17,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237816 2024-11-24T20:47:17,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237816) roll requested 2024-11-24T20:47:17,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237833 2024-11-24T20:47:17,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237833 2024-11-24T20:47:17,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237833) roll requested 2024-11-24T20:47:17,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237842 2024-11-24T20:47:17,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237842 2024-11-24T20:47:17,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237842) roll requested 2024-11-24T20:47:17,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237853 2024-11-24T20:47:17,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237853 2024-11-24T20:47:17,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237853) roll requested 2024-11-24T20:47:17,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237860 2024-11-24T20:47:17,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237860 2024-11-24T20:47:17,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237860) roll requested 2024-11-24T20:47:17,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237871 2024-11-24T20:47:17,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237871 2024-11-24T20:47:17,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237871) roll requested 2024-11-24T20:47:17,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237885 2024-11-24T20:47:17,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237885 2024-11-24T20:47:17,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237885) roll requested 2024-11-24T20:47:17,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237893 2024-11-24T20:47:17,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237893 2024-11-24T20:47:17,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237893) roll requested 2024-11-24T20:47:17,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237899 2024-11-24T20:47:17,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237899 2024-11-24T20:47:17,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237899) roll requested 2024-11-24T20:47:17,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237905 2024-11-24T20:47:17,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237905 2024-11-24T20:47:17,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237905) roll requested 2024-11-24T20:47:17,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237912 2024-11-24T20:47:17,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237912 2024-11-24T20:47:17,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237912) roll requested 2024-11-24T20:47:17,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237920 2024-11-24T20:47:17,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237920 2024-11-24T20:47:17,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237920) roll requested 2024-11-24T20:47:17,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237927 2024-11-24T20:47:17,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237927 2024-11-24T20:47:17,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237927) roll requested 2024-11-24T20:47:17,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237932 2024-11-24T20:47:17,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237932 2024-11-24T20:47:17,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237932) roll requested 2024-11-24T20:47:17,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237940 2024-11-24T20:47:17,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237940 2024-11-24T20:47:17,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237940) roll requested 2024-11-24T20:47:17,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237946 2024-11-24T20:47:17,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237946 2024-11-24T20:47:17,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237946) roll requested 2024-11-24T20:47:17,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237951 2024-11-24T20:47:17,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237951 2024-11-24T20:47:17,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237951) roll requested 2024-11-24T20:47:17,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237960 2024-11-24T20:47:17,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237960 2024-11-24T20:47:17,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237960) roll requested 2024-11-24T20:47:17,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237981 2024-11-24T20:47:17,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:17,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237981 2024-11-24T20:47:17,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:17,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:17,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:17,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:17,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237981) roll requested 2024-11-24T20:47:17,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481237993 2024-11-24T20:47:18,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237993 2024-11-24T20:47:18,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481237993) roll requested 2024-11-24T20:47:18,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238004 2024-11-24T20:47:18,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481237993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238004 2024-11-24T20:47:18,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238004) roll requested 2024-11-24T20:47:18,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238030 2024-11-24T20:47:18,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238030 2024-11-24T20:47:18,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238030) roll requested 2024-11-24T20:47:18,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238037 2024-11-24T20:47:18,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238037 2024-11-24T20:47:18,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238037) roll requested 2024-11-24T20:47:18,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238045 2024-11-24T20:47:18,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238045 2024-11-24T20:47:18,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238045) roll requested 2024-11-24T20:47:18,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238058 2024-11-24T20:47:18,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238058 2024-11-24T20:47:18,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238058) roll requested 2024-11-24T20:47:18,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238073 2024-11-24T20:47:18,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238073 2024-11-24T20:47:18,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238073) roll requested 2024-11-24T20:47:18,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238085 2024-11-24T20:47:18,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238085 2024-11-24T20:47:18,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238085) roll requested 2024-11-24T20:47:18,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238100 2024-11-24T20:47:18,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238100 2024-11-24T20:47:18,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238100) roll requested 2024-11-24T20:47:18,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238126 2024-11-24T20:47:18,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238126 2024-11-24T20:47:18,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238126) roll requested 2024-11-24T20:47:18,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238149 2024-11-24T20:47:18,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238149 2024-11-24T20:47:18,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238149) roll requested 2024-11-24T20:47:18,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238161 2024-11-24T20:47:18,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238161 2024-11-24T20:47:18,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238161) roll requested 2024-11-24T20:47:18,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238189 2024-11-24T20:47:18,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238189 2024-11-24T20:47:18,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238189) roll requested 2024-11-24T20:47:18,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238199 2024-11-24T20:47:18,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238199 2024-11-24T20:47:18,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238199) roll requested 2024-11-24T20:47:18,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238213 2024-11-24T20:47:18,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238213 2024-11-24T20:47:18,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238213) roll requested 2024-11-24T20:47:18,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238227 2024-11-24T20:47:18,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238227 2024-11-24T20:47:18,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238227) roll requested 2024-11-24T20:47:18,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238236 2024-11-24T20:47:18,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238236 2024-11-24T20:47:18,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238236) roll requested 2024-11-24T20:47:18,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238249 2024-11-24T20:47:18,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238249 2024-11-24T20:47:18,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238249) roll requested 2024-11-24T20:47:18,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238257 2024-11-24T20:47:18,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238257 2024-11-24T20:47:18,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238257) roll requested 2024-11-24T20:47:18,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238265 2024-11-24T20:47:18,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238265 2024-11-24T20:47:18,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238265) roll requested 2024-11-24T20:47:18,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238285 2024-11-24T20:47:18,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238285 2024-11-24T20:47:18,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238285) roll requested 2024-11-24T20:47:18,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238293 2024-11-24T20:47:18,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238293 2024-11-24T20:47:18,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238293) roll requested 2024-11-24T20:47:18,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238302 2024-11-24T20:47:18,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238302 2024-11-24T20:47:18,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238302) roll requested 2024-11-24T20:47:18,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238310 2024-11-24T20:47:18,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238310 2024-11-24T20:47:18,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238310) roll requested 2024-11-24T20:47:18,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238320 2024-11-24T20:47:18,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238320 2024-11-24T20:47:18,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238320) roll requested 2024-11-24T20:47:18,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238333 2024-11-24T20:47:18,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238333 2024-11-24T20:47:18,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238333) roll requested 2024-11-24T20:47:18,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238340 2024-11-24T20:47:18,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238340 2024-11-24T20:47:18,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238340) roll requested 2024-11-24T20:47:18,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238348 2024-11-24T20:47:18,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238348 2024-11-24T20:47:18,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238348) roll requested 2024-11-24T20:47:18,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238354 2024-11-24T20:47:18,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238354 2024-11-24T20:47:18,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238354) roll requested 2024-11-24T20:47:18,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238360 2024-11-24T20:47:18,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238360 2024-11-24T20:47:18,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238360) roll requested 2024-11-24T20:47:18,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238369 2024-11-24T20:47:18,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238369 2024-11-24T20:47:18,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238369) roll requested 2024-11-24T20:47:18,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238378 2024-11-24T20:47:18,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238378 2024-11-24T20:47:18,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238378) roll requested 2024-11-24T20:47:18,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238385 2024-11-24T20:47:18,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238385 2024-11-24T20:47:18,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238385) roll requested 2024-11-24T20:47:18,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238393 2024-11-24T20:47:18,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238393 2024-11-24T20:47:18,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238393) roll requested 2024-11-24T20:47:18,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238399 2024-11-24T20:47:18,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238399 2024-11-24T20:47:18,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238399) roll requested 2024-11-24T20:47:18,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238405 2024-11-24T20:47:18,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238405 2024-11-24T20:47:18,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238405) roll requested 2024-11-24T20:47:18,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238416 2024-11-24T20:47:18,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238416 2024-11-24T20:47:18,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238416) roll requested 2024-11-24T20:47:18,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238423 2024-11-24T20:47:18,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238423 2024-11-24T20:47:18,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238423) roll requested 2024-11-24T20:47:18,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238430 2024-11-24T20:47:18,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238430 2024-11-24T20:47:18,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238430) roll requested 2024-11-24T20:47:18,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238442 2024-11-24T20:47:18,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238442 2024-11-24T20:47:18,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238442) roll requested 2024-11-24T20:47:18,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238450 2024-11-24T20:47:18,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238450 2024-11-24T20:47:18,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238450) roll requested 2024-11-24T20:47:18,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238465 2024-11-24T20:47:18,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238465 2024-11-24T20:47:18,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238465) roll requested 2024-11-24T20:47:18,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238473 2024-11-24T20:47:18,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238473 2024-11-24T20:47:18,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238473) roll requested 2024-11-24T20:47:18,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238488 2024-11-24T20:47:18,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238488 2024-11-24T20:47:18,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238488) roll requested 2024-11-24T20:47:18,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238509 2024-11-24T20:47:18,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238509 2024-11-24T20:47:18,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238509) roll requested 2024-11-24T20:47:18,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238525 2024-11-24T20:47:18,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238525 2024-11-24T20:47:18,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238525) roll requested 2024-11-24T20:47:18,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238537 2024-11-24T20:47:18,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238537 2024-11-24T20:47:18,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238537) roll requested 2024-11-24T20:47:18,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238544 2024-11-24T20:47:18,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238544 2024-11-24T20:47:18,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238544) roll requested 2024-11-24T20:47:18,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238565 2024-11-24T20:47:18,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238565 2024-11-24T20:47:18,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238565) roll requested 2024-11-24T20:47:18,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238585 2024-11-24T20:47:18,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238585 2024-11-24T20:47:18,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238585) roll requested 2024-11-24T20:47:18,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238599 2024-11-24T20:47:18,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238599 2024-11-24T20:47:18,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238599) roll requested 2024-11-24T20:47:18,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238609 2024-11-24T20:47:18,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238609 2024-11-24T20:47:18,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238609) roll requested 2024-11-24T20:47:18,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238621 2024-11-24T20:47:18,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238621 2024-11-24T20:47:18,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238621) roll requested 2024-11-24T20:47:18,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238633 2024-11-24T20:47:18,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238633 2024-11-24T20:47:18,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238633) roll requested 2024-11-24T20:47:18,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238645 2024-11-24T20:47:18,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238645 2024-11-24T20:47:18,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238645) roll requested 2024-11-24T20:47:18,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238656 2024-11-24T20:47:18,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238656 2024-11-24T20:47:18,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238656) roll requested 2024-11-24T20:47:18,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238665 2024-11-24T20:47:18,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238665 2024-11-24T20:47:18,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238665) roll requested 2024-11-24T20:47:18,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238673 2024-11-24T20:47:18,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238673 2024-11-24T20:47:18,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238673) roll requested 2024-11-24T20:47:18,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238681 2024-11-24T20:47:18,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238681 2024-11-24T20:47:18,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238681) roll requested 2024-11-24T20:47:18,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238687 2024-11-24T20:47:18,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238687 2024-11-24T20:47:18,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238687) roll requested 2024-11-24T20:47:18,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238695 2024-11-24T20:47:18,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238695 2024-11-24T20:47:18,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238695) roll requested 2024-11-24T20:47:18,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238701 2024-11-24T20:47:18,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238701 2024-11-24T20:47:18,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238701) roll requested 2024-11-24T20:47:18,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238708 2024-11-24T20:47:18,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238708 2024-11-24T20:47:18,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238708) roll requested 2024-11-24T20:47:18,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238714 2024-11-24T20:47:18,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238714 2024-11-24T20:47:18,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238714) roll requested 2024-11-24T20:47:18,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238720 2024-11-24T20:47:18,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238720 2024-11-24T20:47:18,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238720) roll requested 2024-11-24T20:47:18,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238727 2024-11-24T20:47:18,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238727 2024-11-24T20:47:18,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238727) roll requested 2024-11-24T20:47:18,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238740 2024-11-24T20:47:18,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238740 2024-11-24T20:47:18,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238740) roll requested 2024-11-24T20:47:18,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238759 2024-11-24T20:47:18,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238759 2024-11-24T20:47:18,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238759) roll requested 2024-11-24T20:47:18,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238773 2024-11-24T20:47:18,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238773 2024-11-24T20:47:18,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238773) roll requested 2024-11-24T20:47:18,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238785 2024-11-24T20:47:18,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238785 2024-11-24T20:47:18,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238785) roll requested 2024-11-24T20:47:18,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238792 2024-11-24T20:47:18,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238792 2024-11-24T20:47:18,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238792) roll requested 2024-11-24T20:47:18,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238808 2024-11-24T20:47:18,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238808 2024-11-24T20:47:18,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238808) roll requested 2024-11-24T20:47:18,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238819 2024-11-24T20:47:18,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238819 2024-11-24T20:47:18,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238819) roll requested 2024-11-24T20:47:18,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238829 2024-11-24T20:47:18,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238829 2024-11-24T20:47:18,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238829) roll requested 2024-11-24T20:47:18,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238837 2024-11-24T20:47:18,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238837 2024-11-24T20:47:18,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238837) roll requested 2024-11-24T20:47:18,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238849 2024-11-24T20:47:18,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238849 2024-11-24T20:47:18,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238849) roll requested 2024-11-24T20:47:18,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238858 2024-11-24T20:47:18,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238858 2024-11-24T20:47:18,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238858) roll requested 2024-11-24T20:47:18,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238868 2024-11-24T20:47:18,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238868 2024-11-24T20:47:18,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238868) roll requested 2024-11-24T20:47:18,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238880 2024-11-24T20:47:18,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238880 2024-11-24T20:47:18,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238880) roll requested 2024-11-24T20:47:18,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238894 2024-11-24T20:47:18,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238894 2024-11-24T20:47:18,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238894) roll requested 2024-11-24T20:47:18,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238901 2024-11-24T20:47:18,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238901 2024-11-24T20:47:18,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238901) roll requested 2024-11-24T20:47:18,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238908 2024-11-24T20:47:18,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238908 2024-11-24T20:47:18,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238908) roll requested 2024-11-24T20:47:18,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238914 2024-11-24T20:47:18,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238914 2024-11-24T20:47:18,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238914) roll requested 2024-11-24T20:47:18,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238921 2024-11-24T20:47:18,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238921 2024-11-24T20:47:18,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238921) roll requested 2024-11-24T20:47:18,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238937 2024-11-24T20:47:18,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238937 2024-11-24T20:47:18,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238937) roll requested 2024-11-24T20:47:18,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238945 2024-11-24T20:47:18,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238945 2024-11-24T20:47:18,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238945) roll requested 2024-11-24T20:47:18,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238955 2024-11-24T20:47:18,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238955 2024-11-24T20:47:18,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238955) roll requested 2024-11-24T20:47:18,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238964 2024-11-24T20:47:18,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238964 2024-11-24T20:47:18,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238964) roll requested 2024-11-24T20:47:18,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238971 2024-11-24T20:47:18,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238971 2024-11-24T20:47:18,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238971) roll requested 2024-11-24T20:47:18,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238977 2024-11-24T20:47:18,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238977 2024-11-24T20:47:18,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238977) roll requested 2024-11-24T20:47:18,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238985 2024-11-24T20:47:18,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:18,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238985 2024-11-24T20:47:18,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:18,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:18,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:18,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:18,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238985) roll requested 2024-11-24T20:47:18,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481238995 2024-11-24T20:47:19,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238995 2024-11-24T20:47:19,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481238995) roll requested 2024-11-24T20:47:19,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239002 2024-11-24T20:47:19,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481238995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239002 2024-11-24T20:47:19,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239002) roll requested 2024-11-24T20:47:19,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239013 2024-11-24T20:47:19,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239013 2024-11-24T20:47:19,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239013) roll requested 2024-11-24T20:47:19,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239021 2024-11-24T20:47:19,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239021 2024-11-24T20:47:19,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239021) roll requested 2024-11-24T20:47:19,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239030 2024-11-24T20:47:19,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239030 2024-11-24T20:47:19,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239030) roll requested 2024-11-24T20:47:19,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239049 2024-11-24T20:47:19,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239049 2024-11-24T20:47:19,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239049) roll requested 2024-11-24T20:47:19,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239065 2024-11-24T20:47:19,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239065 2024-11-24T20:47:19,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239065) roll requested 2024-11-24T20:47:19,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239074 2024-11-24T20:47:19,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239074 2024-11-24T20:47:19,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,088 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239074) roll requested 2024-11-24T20:47:19,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239088 2024-11-24T20:47:19,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239088 2024-11-24T20:47:19,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239088) roll requested 2024-11-24T20:47:19,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239096 2024-11-24T20:47:19,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239096 2024-11-24T20:47:19,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239096) roll requested 2024-11-24T20:47:19,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239115 2024-11-24T20:47:19,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239115 2024-11-24T20:47:19,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239115) roll requested 2024-11-24T20:47:19,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239124 2024-11-24T20:47:19,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239124 2024-11-24T20:47:19,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239124) roll requested 2024-11-24T20:47:19,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239141 2024-11-24T20:47:19,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239141 2024-11-24T20:47:19,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239141) roll requested 2024-11-24T20:47:19,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239148 2024-11-24T20:47:19,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239148 2024-11-24T20:47:19,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239148) roll requested 2024-11-24T20:47:19,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239157 2024-11-24T20:47:19,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239157 2024-11-24T20:47:19,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,164 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239157) roll requested 2024-11-24T20:47:19,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239164 2024-11-24T20:47:19,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239164 2024-11-24T20:47:19,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239164) roll requested 2024-11-24T20:47:19,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239173 2024-11-24T20:47:19,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239173 2024-11-24T20:47:19,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239173) roll requested 2024-11-24T20:47:19,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239189 2024-11-24T20:47:19,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239189 2024-11-24T20:47:19,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239189) roll requested 2024-11-24T20:47:19,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239196 2024-11-24T20:47:19,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239196 2024-11-24T20:47:19,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239196) roll requested 2024-11-24T20:47:19,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239203 2024-11-24T20:47:19,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239203 2024-11-24T20:47:19,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239203) roll requested 2024-11-24T20:47:19,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239209 2024-11-24T20:47:19,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239209 2024-11-24T20:47:19,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239209) roll requested 2024-11-24T20:47:19,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239217 2024-11-24T20:47:19,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239217 2024-11-24T20:47:19,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239217) roll requested 2024-11-24T20:47:19,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239225 2024-11-24T20:47:19,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239225 2024-11-24T20:47:19,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239225) roll requested 2024-11-24T20:47:19,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239232 2024-11-24T20:47:19,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239232 2024-11-24T20:47:19,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239232) roll requested 2024-11-24T20:47:19,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239250 2024-11-24T20:47:19,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239250 2024-11-24T20:47:19,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239250) roll requested 2024-11-24T20:47:19,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239261 2024-11-24T20:47:19,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239261 2024-11-24T20:47:19,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239261) roll requested 2024-11-24T20:47:19,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239266 2024-11-24T20:47:19,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239266 2024-11-24T20:47:19,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239266) roll requested 2024-11-24T20:47:19,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239277 2024-11-24T20:47:19,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239277 2024-11-24T20:47:19,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239277) roll requested 2024-11-24T20:47:19,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239283 2024-11-24T20:47:19,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239283 2024-11-24T20:47:19,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239283) roll requested 2024-11-24T20:47:19,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239292 2024-11-24T20:47:19,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239292 2024-11-24T20:47:19,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239292) roll requested 2024-11-24T20:47:19,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239301 2024-11-24T20:47:19,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239301 2024-11-24T20:47:19,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239301) roll requested 2024-11-24T20:47:19,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239308 2024-11-24T20:47:19,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239308 2024-11-24T20:47:19,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,315 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239308) roll requested 2024-11-24T20:47:19,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239315 2024-11-24T20:47:19,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239315 2024-11-24T20:47:19,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239315) roll requested 2024-11-24T20:47:19,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239325 2024-11-24T20:47:19,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239325 2024-11-24T20:47:19,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239325) roll requested 2024-11-24T20:47:19,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239331 2024-11-24T20:47:19,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239331 2024-11-24T20:47:19,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239331) roll requested 2024-11-24T20:47:19,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239348 2024-11-24T20:47:19,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239348 2024-11-24T20:47:19,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239348) roll requested 2024-11-24T20:47:19,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239369 2024-11-24T20:47:19,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239369 2024-11-24T20:47:19,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239369) roll requested 2024-11-24T20:47:19,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239375 2024-11-24T20:47:19,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239375 2024-11-24T20:47:19,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239375) roll requested 2024-11-24T20:47:19,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239381 2024-11-24T20:47:19,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239381 2024-11-24T20:47:19,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239381) roll requested 2024-11-24T20:47:19,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239389 2024-11-24T20:47:19,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239389 2024-11-24T20:47:19,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239389) roll requested 2024-11-24T20:47:19,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239396 2024-11-24T20:47:19,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239396 2024-11-24T20:47:19,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239396) roll requested 2024-11-24T20:47:19,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239403 2024-11-24T20:47:19,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239403 2024-11-24T20:47:19,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239403) roll requested 2024-11-24T20:47:19,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239415 2024-11-24T20:47:19,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239415 2024-11-24T20:47:19,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239415) roll requested 2024-11-24T20:47:19,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239422 2024-11-24T20:47:19,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239422 2024-11-24T20:47:19,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239422) roll requested 2024-11-24T20:47:19,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239428 2024-11-24T20:47:19,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239428 2024-11-24T20:47:19,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239428) roll requested 2024-11-24T20:47:19,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239436 2024-11-24T20:47:19,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239436 2024-11-24T20:47:19,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239436) roll requested 2024-11-24T20:47:19,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239443 2024-11-24T20:47:19,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239443 2024-11-24T20:47:19,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239443) roll requested 2024-11-24T20:47:19,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239461 2024-11-24T20:47:19,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239461 2024-11-24T20:47:19,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239461) roll requested 2024-11-24T20:47:19,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239467 2024-11-24T20:47:19,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239467 2024-11-24T20:47:19,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239467) roll requested 2024-11-24T20:47:19,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239477 2024-11-24T20:47:19,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239477 2024-11-24T20:47:19,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239477) roll requested 2024-11-24T20:47:19,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239484 2024-11-24T20:47:19,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239484 2024-11-24T20:47:19,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239484) roll requested 2024-11-24T20:47:19,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239491 2024-11-24T20:47:19,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239491 2024-11-24T20:47:19,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239491) roll requested 2024-11-24T20:47:19,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239499 2024-11-24T20:47:19,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239499 2024-11-24T20:47:19,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239499) roll requested 2024-11-24T20:47:19,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239506 2024-11-24T20:47:19,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239506 2024-11-24T20:47:19,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239506) roll requested 2024-11-24T20:47:19,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239513 2024-11-24T20:47:19,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239513 2024-11-24T20:47:19,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239513) roll requested 2024-11-24T20:47:19,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239521 2024-11-24T20:47:19,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239521 2024-11-24T20:47:19,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239521) roll requested 2024-11-24T20:47:19,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239527 2024-11-24T20:47:19,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239527 2024-11-24T20:47:19,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239527) roll requested 2024-11-24T20:47:19,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239533 2024-11-24T20:47:19,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239533 2024-11-24T20:47:19,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239533) roll requested 2024-11-24T20:47:19,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239547 2024-11-24T20:47:19,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239547 2024-11-24T20:47:19,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239547) roll requested 2024-11-24T20:47:19,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239570 2024-11-24T20:47:19,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239570 2024-11-24T20:47:19,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239570) roll requested 2024-11-24T20:47:19,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239590 2024-11-24T20:47:19,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239590 2024-11-24T20:47:19,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239590) roll requested 2024-11-24T20:47:19,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239596 2024-11-24T20:47:19,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239596 2024-11-24T20:47:19,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239596) roll requested 2024-11-24T20:47:19,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239609 2024-11-24T20:47:19,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239609 2024-11-24T20:47:19,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239609) roll requested 2024-11-24T20:47:19,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239640 2024-11-24T20:47:19,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239640 2024-11-24T20:47:19,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239640) roll requested 2024-11-24T20:47:19,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239652 2024-11-24T20:47:19,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239652 2024-11-24T20:47:19,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239652) roll requested 2024-11-24T20:47:19,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239659 2024-11-24T20:47:19,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239659 2024-11-24T20:47:19,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239659) roll requested 2024-11-24T20:47:19,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239665 2024-11-24T20:47:19,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239665 2024-11-24T20:47:19,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239665) roll requested 2024-11-24T20:47:19,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239673 2024-11-24T20:47:19,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239673 2024-11-24T20:47:19,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239673) roll requested 2024-11-24T20:47:19,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239681 2024-11-24T20:47:19,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239681 2024-11-24T20:47:19,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239681) roll requested 2024-11-24T20:47:19,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239701 2024-11-24T20:47:19,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239701 2024-11-24T20:47:19,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239701) roll requested 2024-11-24T20:47:19,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239717 2024-11-24T20:47:19,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239717 2024-11-24T20:47:19,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239717) roll requested 2024-11-24T20:47:19,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239733 2024-11-24T20:47:19,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239733 2024-11-24T20:47:19,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239733) roll requested 2024-11-24T20:47:19,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239741 2024-11-24T20:47:19,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239741 2024-11-24T20:47:19,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239741) roll requested 2024-11-24T20:47:19,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239748 2024-11-24T20:47:19,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239748 2024-11-24T20:47:19,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239748) roll requested 2024-11-24T20:47:19,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239758 2024-11-24T20:47:19,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239758 2024-11-24T20:47:19,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239758) roll requested 2024-11-24T20:47:19,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239767 2024-11-24T20:47:19,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239767 2024-11-24T20:47:19,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239767) roll requested 2024-11-24T20:47:19,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239782 2024-11-24T20:47:19,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239782 2024-11-24T20:47:19,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239782) roll requested 2024-11-24T20:47:19,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239790 2024-11-24T20:47:19,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239790 2024-11-24T20:47:19,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239790) roll requested 2024-11-24T20:47:19,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239799 2024-11-24T20:47:19,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239799 2024-11-24T20:47:19,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239799) roll requested 2024-11-24T20:47:19,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239811 2024-11-24T20:47:19,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239811 2024-11-24T20:47:19,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239811) roll requested 2024-11-24T20:47:19,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239821 2024-11-24T20:47:19,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239821 2024-11-24T20:47:19,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239821) roll requested 2024-11-24T20:47:19,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239828 2024-11-24T20:47:19,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239828 2024-11-24T20:47:19,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239828) roll requested 2024-11-24T20:47:19,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239836 2024-11-24T20:47:19,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239836 2024-11-24T20:47:19,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239836) roll requested 2024-11-24T20:47:19,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239846 2024-11-24T20:47:19,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239846 2024-11-24T20:47:19,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239846) roll requested 2024-11-24T20:47:19,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239852 2024-11-24T20:47:19,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239852 2024-11-24T20:47:19,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239852) roll requested 2024-11-24T20:47:19,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239860 2024-11-24T20:47:19,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239860 2024-11-24T20:47:19,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239860) roll requested 2024-11-24T20:47:19,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239873 2024-11-24T20:47:19,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239873 2024-11-24T20:47:19,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239873) roll requested 2024-11-24T20:47:19,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239890 2024-11-24T20:47:19,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239890 2024-11-24T20:47:19,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239890) roll requested 2024-11-24T20:47:19,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239901 2024-11-24T20:47:19,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239901 2024-11-24T20:47:19,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239901) roll requested 2024-11-24T20:47:19,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239913 2024-11-24T20:47:19,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239913 2024-11-24T20:47:19,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239913) roll requested 2024-11-24T20:47:19,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239919 2024-11-24T20:47:19,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239919 2024-11-24T20:47:19,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239919) roll requested 2024-11-24T20:47:19,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239932 2024-11-24T20:47:19,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239932 2024-11-24T20:47:19,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239932) roll requested 2024-11-24T20:47:19,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239941 2024-11-24T20:47:19,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239941 2024-11-24T20:47:19,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239941) roll requested 2024-11-24T20:47:19,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239950 2024-11-24T20:47:19,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239950 2024-11-24T20:47:19,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239950) roll requested 2024-11-24T20:47:19,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239957 2024-11-24T20:47:19,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239957 2024-11-24T20:47:19,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239957) roll requested 2024-11-24T20:47:19,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239968 2024-11-24T20:47:19,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:19,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239968 2024-11-24T20:47:19,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:19,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:19,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:19,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:19,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239968) roll requested 2024-11-24T20:47:19,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481239995 2024-11-24T20:47:20,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239995 2024-11-24T20:47:20,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481239995) roll requested 2024-11-24T20:47:20,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240018 2024-11-24T20:47:20,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481239995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240018 2024-11-24T20:47:20,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240018) roll requested 2024-11-24T20:47:20,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240032 2024-11-24T20:47:20,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240032 2024-11-24T20:47:20,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240032) roll requested 2024-11-24T20:47:20,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240041 2024-11-24T20:47:20,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240041 2024-11-24T20:47:20,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240041) roll requested 2024-11-24T20:47:20,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240047 2024-11-24T20:47:20,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240047 2024-11-24T20:47:20,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240047) roll requested 2024-11-24T20:47:20,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240053 2024-11-24T20:47:20,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240053 2024-11-24T20:47:20,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240053) roll requested 2024-11-24T20:47:20,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240063 2024-11-24T20:47:20,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240063 2024-11-24T20:47:20,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240063) roll requested 2024-11-24T20:47:20,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240071 2024-11-24T20:47:20,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240071 2024-11-24T20:47:20,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240071) roll requested 2024-11-24T20:47:20,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240077 2024-11-24T20:47:20,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240077 2024-11-24T20:47:20,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240077) roll requested 2024-11-24T20:47:20,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240086 2024-11-24T20:47:20,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240086 2024-11-24T20:47:20,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240086) roll requested 2024-11-24T20:47:20,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240121 2024-11-24T20:47:20,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240121 2024-11-24T20:47:20,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240121) roll requested 2024-11-24T20:47:20,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240145 2024-11-24T20:47:20,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240145 2024-11-24T20:47:20,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240145) roll requested 2024-11-24T20:47:20,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240155 2024-11-24T20:47:20,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240155 2024-11-24T20:47:20,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240155) roll requested 2024-11-24T20:47:20,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240166 2024-11-24T20:47:20,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240166 2024-11-24T20:47:20,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240166) roll requested 2024-11-24T20:47:20,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240211 2024-11-24T20:47:20,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240211 2024-11-24T20:47:20,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240211) roll requested 2024-11-24T20:47:20,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240237 2024-11-24T20:47:20,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240237 2024-11-24T20:47:20,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240237) roll requested 2024-11-24T20:47:20,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240263 2024-11-24T20:47:20,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240263 2024-11-24T20:47:20,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240263) roll requested 2024-11-24T20:47:20,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240289 2024-11-24T20:47:20,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240289 2024-11-24T20:47:20,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240289) roll requested 2024-11-24T20:47:20,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240305 2024-11-24T20:47:20,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240305 2024-11-24T20:47:20,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240305) roll requested 2024-11-24T20:47:20,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240316 2024-11-24T20:47:20,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240316 2024-11-24T20:47:20,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240316) roll requested 2024-11-24T20:47:20,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240335 2024-11-24T20:47:20,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240335 2024-11-24T20:47:20,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240335) roll requested 2024-11-24T20:47:20,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240356 2024-11-24T20:47:20,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240356 2024-11-24T20:47:20,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240356) roll requested 2024-11-24T20:47:20,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240370 2024-11-24T20:47:20,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240370 2024-11-24T20:47:20,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240370) roll requested 2024-11-24T20:47:20,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240381 2024-11-24T20:47:20,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240381 2024-11-24T20:47:20,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240381) roll requested 2024-11-24T20:47:20,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240393 2024-11-24T20:47:20,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240393 2024-11-24T20:47:20,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240393) roll requested 2024-11-24T20:47:20,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240405 2024-11-24T20:47:20,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240405 2024-11-24T20:47:20,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240405) roll requested 2024-11-24T20:47:20,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240415 2024-11-24T20:47:20,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240415 2024-11-24T20:47:20,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240415) roll requested 2024-11-24T20:47:20,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240429 2024-11-24T20:47:20,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240429 2024-11-24T20:47:20,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240429) roll requested 2024-11-24T20:47:20,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240441 2024-11-24T20:47:20,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240441 2024-11-24T20:47:20,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240441) roll requested 2024-11-24T20:47:20,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240451 2024-11-24T20:47:20,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240451 2024-11-24T20:47:20,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240451) roll requested 2024-11-24T20:47:20,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240460 2024-11-24T20:47:20,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240460 2024-11-24T20:47:20,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240460) roll requested 2024-11-24T20:47:20,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240470 2024-11-24T20:47:20,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240470 2024-11-24T20:47:20,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240470) roll requested 2024-11-24T20:47:20,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240476 2024-11-24T20:47:20,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240476 2024-11-24T20:47:20,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240476) roll requested 2024-11-24T20:47:20,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240485 2024-11-24T20:47:20,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240485 2024-11-24T20:47:20,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240485) roll requested 2024-11-24T20:47:20,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240493 2024-11-24T20:47:20,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240493 2024-11-24T20:47:20,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240493) roll requested 2024-11-24T20:47:20,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240501 2024-11-24T20:47:20,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240501 2024-11-24T20:47:20,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240501) roll requested 2024-11-24T20:47:20,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240508 2024-11-24T20:47:20,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240508 2024-11-24T20:47:20,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240508) roll requested 2024-11-24T20:47:20,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240514 2024-11-24T20:47:20,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240514 2024-11-24T20:47:20,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240514) roll requested 2024-11-24T20:47:20,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240522 2024-11-24T20:47:20,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240522 2024-11-24T20:47:20,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240522) roll requested 2024-11-24T20:47:20,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240530 2024-11-24T20:47:20,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240530 2024-11-24T20:47:20,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240530) roll requested 2024-11-24T20:47:20,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240538 2024-11-24T20:47:20,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240538 2024-11-24T20:47:20,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240538) roll requested 2024-11-24T20:47:20,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240544 2024-11-24T20:47:20,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240544 2024-11-24T20:47:20,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240544) roll requested 2024-11-24T20:47:20,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240551 2024-11-24T20:47:20,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240551 2024-11-24T20:47:20,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240551) roll requested 2024-11-24T20:47:20,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240557 2024-11-24T20:47:20,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240557 2024-11-24T20:47:20,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240557) roll requested 2024-11-24T20:47:20,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240563 2024-11-24T20:47:20,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240563 2024-11-24T20:47:20,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240563) roll requested 2024-11-24T20:47:20,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240572 2024-11-24T20:47:20,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240572 2024-11-24T20:47:20,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240572) roll requested 2024-11-24T20:47:20,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240578 2024-11-24T20:47:20,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240578 2024-11-24T20:47:20,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240578) roll requested 2024-11-24T20:47:20,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240589 2024-11-24T20:47:20,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240589 2024-11-24T20:47:20,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240589) roll requested 2024-11-24T20:47:20,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240597 2024-11-24T20:47:20,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240597 2024-11-24T20:47:20,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240597) roll requested 2024-11-24T20:47:20,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240609 2024-11-24T20:47:20,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240609 2024-11-24T20:47:20,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240609) roll requested 2024-11-24T20:47:20,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240620 2024-11-24T20:47:20,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240620 2024-11-24T20:47:20,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240620) roll requested 2024-11-24T20:47:20,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240626 2024-11-24T20:47:20,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240626 2024-11-24T20:47:20,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240626) roll requested 2024-11-24T20:47:20,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240636 2024-11-24T20:47:20,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240636 2024-11-24T20:47:20,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240636) roll requested 2024-11-24T20:47:20,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240660 2024-11-24T20:47:20,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240660 2024-11-24T20:47:20,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240660) roll requested 2024-11-24T20:47:20,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240678 2024-11-24T20:47:20,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240678 2024-11-24T20:47:20,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240678) roll requested 2024-11-24T20:47:20,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240689 2024-11-24T20:47:20,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240689 2024-11-24T20:47:20,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240689) roll requested 2024-11-24T20:47:20,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240698 2024-11-24T20:47:20,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240698 2024-11-24T20:47:20,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,711 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,711 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240698) roll requested 2024-11-24T20:47:20,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240711 2024-11-24T20:47:20,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240711 2024-11-24T20:47:20,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240711) roll requested 2024-11-24T20:47:20,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240720 2024-11-24T20:47:20,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240711 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240720 2024-11-24T20:47:20,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240720) roll requested 2024-11-24T20:47:20,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240726 2024-11-24T20:47:20,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240726 2024-11-24T20:47:20,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240726) roll requested 2024-11-24T20:47:20,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240736 2024-11-24T20:47:20,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240736 2024-11-24T20:47:20,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240736) roll requested 2024-11-24T20:47:20,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240743 2024-11-24T20:47:20,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240743 2024-11-24T20:47:20,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240743) roll requested 2024-11-24T20:47:20,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240749 2024-11-24T20:47:20,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240749 2024-11-24T20:47:20,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240749) roll requested 2024-11-24T20:47:20,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240766 2024-11-24T20:47:20,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240766 2024-11-24T20:47:20,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240766) roll requested 2024-11-24T20:47:20,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240773 2024-11-24T20:47:20,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240773 2024-11-24T20:47:20,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240773) roll requested 2024-11-24T20:47:20,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240780 2024-11-24T20:47:20,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240780 2024-11-24T20:47:20,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240780) roll requested 2024-11-24T20:47:20,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240789 2024-11-24T20:47:20,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240789 2024-11-24T20:47:20,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240789) roll requested 2024-11-24T20:47:20,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240798 2024-11-24T20:47:20,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240798 2024-11-24T20:47:20,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240798) roll requested 2024-11-24T20:47:20,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240809 2024-11-24T20:47:20,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240809 2024-11-24T20:47:20,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240809) roll requested 2024-11-24T20:47:20,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240818 2024-11-24T20:47:20,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240818 2024-11-24T20:47:20,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240818) roll requested 2024-11-24T20:47:20,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240825 2024-11-24T20:47:20,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240825 2024-11-24T20:47:20,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240825) roll requested 2024-11-24T20:47:20,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240843 2024-11-24T20:47:20,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240843 2024-11-24T20:47:20,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240843) roll requested 2024-11-24T20:47:20,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240852 2024-11-24T20:47:20,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240852 2024-11-24T20:47:20,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240852) roll requested 2024-11-24T20:47:20,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240912 2024-11-24T20:47:20,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240912 2024-11-24T20:47:20,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240912) roll requested 2024-11-24T20:47:20,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240932 2024-11-24T20:47:20,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240932 2024-11-24T20:47:20,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240932) roll requested 2024-11-24T20:47:20,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240951 2024-11-24T20:47:20,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240951 2024-11-24T20:47:20,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240951) roll requested 2024-11-24T20:47:20,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240969 2024-11-24T20:47:20,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240969 2024-11-24T20:47:20,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240969) roll requested 2024-11-24T20:47:20,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240976 2024-11-24T20:47:20,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240976 2024-11-24T20:47:20,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:20,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:20,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:20,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:20,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240976) roll requested 2024-11-24T20:47:20,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481240984 2024-11-24T20:47:20,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:20,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240984 2024-11-24T20:47:21,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481240984) roll requested 2024-11-24T20:47:21,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241001 2024-11-24T20:47:21,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481240984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241001 2024-11-24T20:47:21,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241001) roll requested 2024-11-24T20:47:21,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241017 2024-11-24T20:47:21,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241017 2024-11-24T20:47:21,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241017) roll requested 2024-11-24T20:47:21,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241023 2024-11-24T20:47:21,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241023 2024-11-24T20:47:21,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241023) roll requested 2024-11-24T20:47:21,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241029 2024-11-24T20:47:21,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241029 2024-11-24T20:47:21,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241029) roll requested 2024-11-24T20:47:21,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241037 2024-11-24T20:47:21,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241037 2024-11-24T20:47:21,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241037) roll requested 2024-11-24T20:47:21,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241045 2024-11-24T20:47:21,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241045 2024-11-24T20:47:21,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241045) roll requested 2024-11-24T20:47:21,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241054 2024-11-24T20:47:21,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241054 2024-11-24T20:47:21,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241054) roll requested 2024-11-24T20:47:21,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241061 2024-11-24T20:47:21,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241061 2024-11-24T20:47:21,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241061) roll requested 2024-11-24T20:47:21,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241073 2024-11-24T20:47:21,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241073 2024-11-24T20:47:21,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241073) roll requested 2024-11-24T20:47:21,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241087 2024-11-24T20:47:21,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241087 2024-11-24T20:47:21,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241087) roll requested 2024-11-24T20:47:21,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241111 2024-11-24T20:47:21,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241111 2024-11-24T20:47:21,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241111) roll requested 2024-11-24T20:47:21,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241121 2024-11-24T20:47:21,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241121 2024-11-24T20:47:21,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241121) roll requested 2024-11-24T20:47:21,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241130 2024-11-24T20:47:21,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241130 2024-11-24T20:47:21,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241130) roll requested 2024-11-24T20:47:21,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241141 2024-11-24T20:47:21,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241141 2024-11-24T20:47:21,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241141) roll requested 2024-11-24T20:47:21,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241147 2024-11-24T20:47:21,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241147 2024-11-24T20:47:21,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241147) roll requested 2024-11-24T20:47:21,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241154 2024-11-24T20:47:21,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241154 2024-11-24T20:47:21,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241154) roll requested 2024-11-24T20:47:21,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241167 2024-11-24T20:47:21,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241167 2024-11-24T20:47:21,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241167) roll requested 2024-11-24T20:47:21,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241179 2024-11-24T20:47:21,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241179 2024-11-24T20:47:21,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241179) roll requested 2024-11-24T20:47:21,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241188 2024-11-24T20:47:21,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241188 2024-11-24T20:47:21,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241188) roll requested 2024-11-24T20:47:21,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241197 2024-11-24T20:47:21,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241197 2024-11-24T20:47:21,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241197) roll requested 2024-11-24T20:47:21,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241213 2024-11-24T20:47:21,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241213 2024-11-24T20:47:21,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241213) roll requested 2024-11-24T20:47:21,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241225 2024-11-24T20:47:21,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241225 2024-11-24T20:47:21,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241225) roll requested 2024-11-24T20:47:21,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241233 2024-11-24T20:47:21,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241233 2024-11-24T20:47:21,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241233) roll requested 2024-11-24T20:47:21,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241245 2024-11-24T20:47:21,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241245 2024-11-24T20:47:21,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241245) roll requested 2024-11-24T20:47:21,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241261 2024-11-24T20:47:21,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241261 2024-11-24T20:47:21,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241261) roll requested 2024-11-24T20:47:21,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241277 2024-11-24T20:47:21,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241277 2024-11-24T20:47:21,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241277) roll requested 2024-11-24T20:47:21,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241285 2024-11-24T20:47:21,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241285 2024-11-24T20:47:21,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241285) roll requested 2024-11-24T20:47:21,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241294 2024-11-24T20:47:21,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241294 2024-11-24T20:47:21,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241294) roll requested 2024-11-24T20:47:21,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241307 2024-11-24T20:47:21,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241307 2024-11-24T20:47:21,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241307) roll requested 2024-11-24T20:47:21,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241318 2024-11-24T20:47:21,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241318 2024-11-24T20:47:21,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241318) roll requested 2024-11-24T20:47:21,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241327 2024-11-24T20:47:21,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241327 2024-11-24T20:47:21,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241327) roll requested 2024-11-24T20:47:21,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241333 2024-11-24T20:47:21,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241333 2024-11-24T20:47:21,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241333) roll requested 2024-11-24T20:47:21,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241344 2024-11-24T20:47:21,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241344 2024-11-24T20:47:21,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241344) roll requested 2024-11-24T20:47:21,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241352 2024-11-24T20:47:21,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241352 2024-11-24T20:47:21,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241344 is not closed yet, will try archiving it next time 2024-11-24T20:47:21,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241352) roll requested 2024-11-24T20:47:21,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241360 2024-11-24T20:47:21,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241360 2024-11-24T20:47:21,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241360) roll requested 2024-11-24T20:47:21,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241369 2024-11-24T20:47:21,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241369 2024-11-24T20:47:21,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241369) roll requested 2024-11-24T20:47:21,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241381 2024-11-24T20:47:21,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241381 2024-11-24T20:47:21,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241381) roll requested 2024-11-24T20:47:21,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241389 2024-11-24T20:47:21,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241389 2024-11-24T20:47:21,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241389) roll requested 2024-11-24T20:47:21,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241401 2024-11-24T20:47:21,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241401 2024-11-24T20:47:21,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241401) roll requested 2024-11-24T20:47:21,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241412 2024-11-24T20:47:21,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241412 2024-11-24T20:47:21,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241412) roll requested 2024-11-24T20:47:21,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241425 2024-11-24T20:47:21,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241425 2024-11-24T20:47:21,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241425) roll requested 2024-11-24T20:47:21,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241432 2024-11-24T20:47:21,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241432 2024-11-24T20:47:21,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241432) roll requested 2024-11-24T20:47:21,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241441 2024-11-24T20:47:21,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241441 2024-11-24T20:47:21,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241441) roll requested 2024-11-24T20:47:21,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241450 2024-11-24T20:47:21,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241450 2024-11-24T20:47:21,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241450) roll requested 2024-11-24T20:47:21,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241456 2024-11-24T20:47:21,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241456 2024-11-24T20:47:21,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241456) roll requested 2024-11-24T20:47:21,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241462 2024-11-24T20:47:21,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241462 2024-11-24T20:47:21,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241462) roll requested 2024-11-24T20:47:21,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241477 2024-11-24T20:47:21,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241477 2024-11-24T20:47:21,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241477) roll requested 2024-11-24T20:47:21,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241486 2024-11-24T20:47:21,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241486 2024-11-24T20:47:21,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241486) roll requested 2024-11-24T20:47:21,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241493 2024-11-24T20:47:21,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241493 2024-11-24T20:47:21,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241493) roll requested 2024-11-24T20:47:21,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241501 2024-11-24T20:47:21,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241501 2024-11-24T20:47:21,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241501) roll requested 2024-11-24T20:47:21,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241509 2024-11-24T20:47:21,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241509 2024-11-24T20:47:21,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241509) roll requested 2024-11-24T20:47:21,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241517 2024-11-24T20:47:21,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241517 2024-11-24T20:47:21,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241517) roll requested 2024-11-24T20:47:21,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241529 2024-11-24T20:47:21,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241529 2024-11-24T20:47:21,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241529) roll requested 2024-11-24T20:47:21,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241540 2024-11-24T20:47:21,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241540 2024-11-24T20:47:21,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241540) roll requested 2024-11-24T20:47:21,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241557 2024-11-24T20:47:21,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241557 2024-11-24T20:47:21,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241557) roll requested 2024-11-24T20:47:21,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241570 2024-11-24T20:47:21,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241570 2024-11-24T20:47:21,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241570) roll requested 2024-11-24T20:47:21,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241584 2024-11-24T20:47:21,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241584 2024-11-24T20:47:21,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241584) roll requested 2024-11-24T20:47:21,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241596 2024-11-24T20:47:21,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241596 2024-11-24T20:47:21,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241596) roll requested 2024-11-24T20:47:21,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241605 2024-11-24T20:47:21,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241605 2024-11-24T20:47:21,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241605) roll requested 2024-11-24T20:47:21,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241611 2024-11-24T20:47:21,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241611 2024-11-24T20:47:21,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241611) roll requested 2024-11-24T20:47:21,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241620 2024-11-24T20:47:21,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241620 2024-11-24T20:47:21,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241620) roll requested 2024-11-24T20:47:21,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241628 2024-11-24T20:47:21,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241628 2024-11-24T20:47:21,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241628) roll requested 2024-11-24T20:47:21,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241635 2024-11-24T20:47:21,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241635 2024-11-24T20:47:21,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241635) roll requested 2024-11-24T20:47:21,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241645 2024-11-24T20:47:21,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241645 2024-11-24T20:47:21,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241645) roll requested 2024-11-24T20:47:21,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241653 2024-11-24T20:47:21,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241653 2024-11-24T20:47:21,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241653) roll requested 2024-11-24T20:47:21,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241660 2024-11-24T20:47:21,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241660 2024-11-24T20:47:21,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241660) roll requested 2024-11-24T20:47:21,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241667 2024-11-24T20:47:21,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241667 2024-11-24T20:47:21,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241667) roll requested 2024-11-24T20:47:21,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241674 2024-11-24T20:47:21,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241674 2024-11-24T20:47:21,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241674) roll requested 2024-11-24T20:47:21,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241681 2024-11-24T20:47:21,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241681 2024-11-24T20:47:21,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241681) roll requested 2024-11-24T20:47:21,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241689 2024-11-24T20:47:21,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241689 2024-11-24T20:47:21,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241689) roll requested 2024-11-24T20:47:21,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241697 2024-11-24T20:47:21,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241697 2024-11-24T20:47:21,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241697) roll requested 2024-11-24T20:47:21,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241704 2024-11-24T20:47:21,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241704 2024-11-24T20:47:21,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241704) roll requested 2024-11-24T20:47:21,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241712 2024-11-24T20:47:21,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241712 2024-11-24T20:47:21,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241712) roll requested 2024-11-24T20:47:21,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241721 2024-11-24T20:47:21,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241721 2024-11-24T20:47:21,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241721) roll requested 2024-11-24T20:47:21,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241727 2024-11-24T20:47:21,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241727 2024-11-24T20:47:21,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241727) roll requested 2024-11-24T20:47:21,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241737 2024-11-24T20:47:21,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241737 2024-11-24T20:47:21,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241737) roll requested 2024-11-24T20:47:21,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241746 2024-11-24T20:47:21,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241746 2024-11-24T20:47:21,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241746) roll requested 2024-11-24T20:47:21,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241754 2024-11-24T20:47:21,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241754 2024-11-24T20:47:21,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241754) roll requested 2024-11-24T20:47:21,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241761 2024-11-24T20:47:21,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241761 2024-11-24T20:47:21,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241761) roll requested 2024-11-24T20:47:21,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241767 2024-11-24T20:47:21,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241767 2024-11-24T20:47:21,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241767) roll requested 2024-11-24T20:47:21,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241774 2024-11-24T20:47:21,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241774 2024-11-24T20:47:21,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241774) roll requested 2024-11-24T20:47:21,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241781 2024-11-24T20:47:21,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241781 2024-11-24T20:47:21,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241781) roll requested 2024-11-24T20:47:21,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241788 2024-11-24T20:47:21,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241788 2024-11-24T20:47:21,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241788) roll requested 2024-11-24T20:47:21,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241797 2024-11-24T20:47:21,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241797 2024-11-24T20:47:21,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241797) roll requested 2024-11-24T20:47:21,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241804 2024-11-24T20:47:21,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241804 2024-11-24T20:47:21,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241804) roll requested 2024-11-24T20:47:21,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241811 2024-11-24T20:47:21,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241811 2024-11-24T20:47:21,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241811) roll requested 2024-11-24T20:47:21,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241822 2024-11-24T20:47:21,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241822 2024-11-24T20:47:21,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241822) roll requested 2024-11-24T20:47:21,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241828 2024-11-24T20:47:21,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241828 2024-11-24T20:47:21,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241828) roll requested 2024-11-24T20:47:21,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241835 2024-11-24T20:47:21,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241835 2024-11-24T20:47:21,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241835) roll requested 2024-11-24T20:47:21,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241842 2024-11-24T20:47:21,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241842 2024-11-24T20:47:21,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241842) roll requested 2024-11-24T20:47:21,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241850 2024-11-24T20:47:21,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241850 2024-11-24T20:47:21,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241850) roll requested 2024-11-24T20:47:21,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241856 2024-11-24T20:47:21,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241856 2024-11-24T20:47:21,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241856) roll requested 2024-11-24T20:47:21,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241861 2024-11-24T20:47:21,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241861 2024-11-24T20:47:21,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241861) roll requested 2024-11-24T20:47:21,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241867 2024-11-24T20:47:21,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241867 2024-11-24T20:47:21,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241867) roll requested 2024-11-24T20:47:21,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241874 2024-11-24T20:47:21,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241874 2024-11-24T20:47:21,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241874) roll requested 2024-11-24T20:47:21,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241882 2024-11-24T20:47:21,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241882 2024-11-24T20:47:21,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241882) roll requested 2024-11-24T20:47:21,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241891 2024-11-24T20:47:21,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241891 2024-11-24T20:47:21,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241891) roll requested 2024-11-24T20:47:21,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241900 2024-11-24T20:47:21,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241900 2024-11-24T20:47:21,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241900) roll requested 2024-11-24T20:47:21,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241909 2024-11-24T20:47:21,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241909 2024-11-24T20:47:21,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241909) roll requested 2024-11-24T20:47:21,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241918 2024-11-24T20:47:21,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241918 2024-11-24T20:47:21,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241918) roll requested 2024-11-24T20:47:21,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241928 2024-11-24T20:47:21,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241928 2024-11-24T20:47:21,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241928) roll requested 2024-11-24T20:47:21,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241935 2024-11-24T20:47:21,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241935 2024-11-24T20:47:21,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241935) roll requested 2024-11-24T20:47:21,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241940 2024-11-24T20:47:21,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241940 2024-11-24T20:47:21,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241940) roll requested 2024-11-24T20:47:21,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241946 2024-11-24T20:47:21,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241946 2024-11-24T20:47:21,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241946) roll requested 2024-11-24T20:47:21,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241952 2024-11-24T20:47:21,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241952 2024-11-24T20:47:21,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241952) roll requested 2024-11-24T20:47:21,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241959 2024-11-24T20:47:21,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241959 2024-11-24T20:47:21,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241959) roll requested 2024-11-24T20:47:21,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241967 2024-11-24T20:47:21,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241967 2024-11-24T20:47:21,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241967) roll requested 2024-11-24T20:47:21,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241976 2024-11-24T20:47:21,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241976 2024-11-24T20:47:21,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241976) roll requested 2024-11-24T20:47:21,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241985 2024-11-24T20:47:21,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241985 2024-11-24T20:47:21,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241985) roll requested 2024-11-24T20:47:21,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241993 2024-11-24T20:47:21,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:21,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241993 2024-11-24T20:47:21,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:21,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:21,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:21,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:21,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241993) roll requested 2024-11-24T20:47:21,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481241999 2024-11-24T20:47:22,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241999 2024-11-24T20:47:22,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481241999) roll requested 2024-11-24T20:47:22,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242006 2024-11-24T20:47:22,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481241999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242006 2024-11-24T20:47:22,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242006) roll requested 2024-11-24T20:47:22,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242013 2024-11-24T20:47:22,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242013 2024-11-24T20:47:22,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242013) roll requested 2024-11-24T20:47:22,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242020 2024-11-24T20:47:22,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242020 2024-11-24T20:47:22,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242020) roll requested 2024-11-24T20:47:22,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242027 2024-11-24T20:47:22,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242027 2024-11-24T20:47:22,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242027) roll requested 2024-11-24T20:47:22,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242036 2024-11-24T20:47:22,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242036 2024-11-24T20:47:22,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242036) roll requested 2024-11-24T20:47:22,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242045 2024-11-24T20:47:22,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242045 2024-11-24T20:47:22,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242045) roll requested 2024-11-24T20:47:22,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242054 2024-11-24T20:47:22,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242054 2024-11-24T20:47:22,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242054) roll requested 2024-11-24T20:47:22,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242063 2024-11-24T20:47:22,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242063 2024-11-24T20:47:22,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242063) roll requested 2024-11-24T20:47:22,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242071 2024-11-24T20:47:22,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242071 2024-11-24T20:47:22,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242071) roll requested 2024-11-24T20:47:22,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242081 2024-11-24T20:47:22,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242081 2024-11-24T20:47:22,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242081) roll requested 2024-11-24T20:47:22,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242090 2024-11-24T20:47:22,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242090 2024-11-24T20:47:22,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242090) roll requested 2024-11-24T20:47:22,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242098 2024-11-24T20:47:22,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242098 2024-11-24T20:47:22,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242098) roll requested 2024-11-24T20:47:22,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242109 2024-11-24T20:47:22,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242109 2024-11-24T20:47:22,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242109) roll requested 2024-11-24T20:47:22,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242122 2024-11-24T20:47:22,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242122 2024-11-24T20:47:22,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242122) roll requested 2024-11-24T20:47:22,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242130 2024-11-24T20:47:22,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242130 2024-11-24T20:47:22,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242130) roll requested 2024-11-24T20:47:22,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242138 2024-11-24T20:47:22,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242138 2024-11-24T20:47:22,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242138) roll requested 2024-11-24T20:47:22,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242148 2024-11-24T20:47:22,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242148 2024-11-24T20:47:22,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,158 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242148) roll requested 2024-11-24T20:47:22,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242158 2024-11-24T20:47:22,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242158 2024-11-24T20:47:22,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242158) roll requested 2024-11-24T20:47:22,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242167 2024-11-24T20:47:22,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242167 2024-11-24T20:47:22,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242167) roll requested 2024-11-24T20:47:22,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242178 2024-11-24T20:47:22,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242178 2024-11-24T20:47:22,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242178) roll requested 2024-11-24T20:47:22,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242185 2024-11-24T20:47:22,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242185 2024-11-24T20:47:22,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242185) roll requested 2024-11-24T20:47:22,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242192 2024-11-24T20:47:22,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242192 2024-11-24T20:47:22,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242192) roll requested 2024-11-24T20:47:22,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242198 2024-11-24T20:47:22,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242198 2024-11-24T20:47:22,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242198) roll requested 2024-11-24T20:47:22,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242203 2024-11-24T20:47:22,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242203 2024-11-24T20:47:22,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242203) roll requested 2024-11-24T20:47:22,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242209 2024-11-24T20:47:22,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242209 2024-11-24T20:47:22,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242209) roll requested 2024-11-24T20:47:22,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242214 2024-11-24T20:47:22,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242214 2024-11-24T20:47:22,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242214) roll requested 2024-11-24T20:47:22,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242220 2024-11-24T20:47:22,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242220 2024-11-24T20:47:22,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242220) roll requested 2024-11-24T20:47:22,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242227 2024-11-24T20:47:22,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242227 2024-11-24T20:47:22,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242227) roll requested 2024-11-24T20:47:22,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242235 2024-11-24T20:47:22,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242235 2024-11-24T20:47:22,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242235) roll requested 2024-11-24T20:47:22,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242244 2024-11-24T20:47:22,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242244 2024-11-24T20:47:22,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242244) roll requested 2024-11-24T20:47:22,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242253 2024-11-24T20:47:22,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242253 2024-11-24T20:47:22,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242253) roll requested 2024-11-24T20:47:22,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242259 2024-11-24T20:47:22,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242259 2024-11-24T20:47:22,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242259) roll requested 2024-11-24T20:47:22,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242265 2024-11-24T20:47:22,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242265 2024-11-24T20:47:22,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242265) roll requested 2024-11-24T20:47:22,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242272 2024-11-24T20:47:22,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242272 2024-11-24T20:47:22,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242272) roll requested 2024-11-24T20:47:22,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242279 2024-11-24T20:47:22,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242279 2024-11-24T20:47:22,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242279) roll requested 2024-11-24T20:47:22,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242288 2024-11-24T20:47:22,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242288 2024-11-24T20:47:22,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242288) roll requested 2024-11-24T20:47:22,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242296 2024-11-24T20:47:22,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242296 2024-11-24T20:47:22,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242296) roll requested 2024-11-24T20:47:22,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242303 2024-11-24T20:47:22,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242303 2024-11-24T20:47:22,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242303) roll requested 2024-11-24T20:47:22,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242309 2024-11-24T20:47:22,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242309 2024-11-24T20:47:22,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242309) roll requested 2024-11-24T20:47:22,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242317 2024-11-24T20:47:22,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242317 2024-11-24T20:47:22,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242317) roll requested 2024-11-24T20:47:22,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242325 2024-11-24T20:47:22,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242325 2024-11-24T20:47:22,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242325) roll requested 2024-11-24T20:47:22,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242334 2024-11-24T20:47:22,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242334 2024-11-24T20:47:22,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242334) roll requested 2024-11-24T20:47:22,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242344 2024-11-24T20:47:22,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242344 2024-11-24T20:47:22,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242344) roll requested 2024-11-24T20:47:22,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242355 2024-11-24T20:47:22,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242355 2024-11-24T20:47:22,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242355) roll requested 2024-11-24T20:47:22,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242363 2024-11-24T20:47:22,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242363 2024-11-24T20:47:22,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242363) roll requested 2024-11-24T20:47:22,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242369 2024-11-24T20:47:22,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242369 2024-11-24T20:47:22,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242369) roll requested 2024-11-24T20:47:22,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242374 2024-11-24T20:47:22,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242374 2024-11-24T20:47:22,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242374) roll requested 2024-11-24T20:47:22,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242380 2024-11-24T20:47:22,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242380 2024-11-24T20:47:22,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242380) roll requested 2024-11-24T20:47:22,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242385 2024-11-24T20:47:22,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242385 2024-11-24T20:47:22,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242385) roll requested 2024-11-24T20:47:22,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242394 2024-11-24T20:47:22,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242394 2024-11-24T20:47:22,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242394) roll requested 2024-11-24T20:47:22,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242401 2024-11-24T20:47:22,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242401 2024-11-24T20:47:22,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242401) roll requested 2024-11-24T20:47:22,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242411 2024-11-24T20:47:22,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242411 2024-11-24T20:47:22,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242411) roll requested 2024-11-24T20:47:22,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242421 2024-11-24T20:47:22,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242421 2024-11-24T20:47:22,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242421) roll requested 2024-11-24T20:47:22,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242427 2024-11-24T20:47:22,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242427 2024-11-24T20:47:22,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242427) roll requested 2024-11-24T20:47:22,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242434 2024-11-24T20:47:22,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242434 2024-11-24T20:47:22,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242434) roll requested 2024-11-24T20:47:22,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242441 2024-11-24T20:47:22,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242441 2024-11-24T20:47:22,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242441) roll requested 2024-11-24T20:47:22,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242450 2024-11-24T20:47:22,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242450 2024-11-24T20:47:22,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242450) roll requested 2024-11-24T20:47:22,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242458 2024-11-24T20:47:22,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242458 2024-11-24T20:47:22,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242458) roll requested 2024-11-24T20:47:22,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242468 2024-11-24T20:47:22,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242468 2024-11-24T20:47:22,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242468) roll requested 2024-11-24T20:47:22,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242474 2024-11-24T20:47:22,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242474 2024-11-24T20:47:22,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242474) roll requested 2024-11-24T20:47:22,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242480 2024-11-24T20:47:22,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242480 2024-11-24T20:47:22,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242480) roll requested 2024-11-24T20:47:22,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242489 2024-11-24T20:47:22,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242489 2024-11-24T20:47:22,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242489) roll requested 2024-11-24T20:47:22,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242495 2024-11-24T20:47:22,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242495 2024-11-24T20:47:22,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242495) roll requested 2024-11-24T20:47:22,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242501 2024-11-24T20:47:22,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242501 2024-11-24T20:47:22,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242501) roll requested 2024-11-24T20:47:22,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242507 2024-11-24T20:47:22,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242507 2024-11-24T20:47:22,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242507) roll requested 2024-11-24T20:47:22,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242514 2024-11-24T20:47:22,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242514 2024-11-24T20:47:22,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242514) roll requested 2024-11-24T20:47:22,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242521 2024-11-24T20:47:22,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242521 2024-11-24T20:47:22,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242521) roll requested 2024-11-24T20:47:22,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242526 2024-11-24T20:47:22,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242526 2024-11-24T20:47:22,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242526) roll requested 2024-11-24T20:47:22,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242532 2024-11-24T20:47:22,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242532 2024-11-24T20:47:22,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242532) roll requested 2024-11-24T20:47:22,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242538 2024-11-24T20:47:22,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242538 2024-11-24T20:47:22,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242538) roll requested 2024-11-24T20:47:22,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242543 2024-11-24T20:47:22,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242543 2024-11-24T20:47:22,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242543) roll requested 2024-11-24T20:47:22,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242550 2024-11-24T20:47:22,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242550 2024-11-24T20:47:22,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242550) roll requested 2024-11-24T20:47:22,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242557 2024-11-24T20:47:22,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242557 2024-11-24T20:47:22,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242557) roll requested 2024-11-24T20:47:22,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242564 2024-11-24T20:47:22,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242564 2024-11-24T20:47:22,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242564) roll requested 2024-11-24T20:47:22,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242572 2024-11-24T20:47:22,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242572 2024-11-24T20:47:22,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242572) roll requested 2024-11-24T20:47:22,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242578 2024-11-24T20:47:22,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242578 2024-11-24T20:47:22,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242578) roll requested 2024-11-24T20:47:22,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242585 2024-11-24T20:47:22,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242585 2024-11-24T20:47:22,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242585) roll requested 2024-11-24T20:47:22,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242598 2024-11-24T20:47:22,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242598 2024-11-24T20:47:22,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242598) roll requested 2024-11-24T20:47:22,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242605 2024-11-24T20:47:22,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242605 2024-11-24T20:47:22,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242605) roll requested 2024-11-24T20:47:22,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242613 2024-11-24T20:47:22,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242613 2024-11-24T20:47:22,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242613) roll requested 2024-11-24T20:47:22,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242619 2024-11-24T20:47:22,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242619 2024-11-24T20:47:22,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242619) roll requested 2024-11-24T20:47:22,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242624 2024-11-24T20:47:22,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242624 2024-11-24T20:47:22,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242624) roll requested 2024-11-24T20:47:22,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242629 2024-11-24T20:47:22,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242629 2024-11-24T20:47:22,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242629) roll requested 2024-11-24T20:47:22,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242635 2024-11-24T20:47:22,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242635 2024-11-24T20:47:22,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242635) roll requested 2024-11-24T20:47:22,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242642 2024-11-24T20:47:22,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242642 2024-11-24T20:47:22,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242642) roll requested 2024-11-24T20:47:22,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242649 2024-11-24T20:47:22,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242649 2024-11-24T20:47:22,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242649) roll requested 2024-11-24T20:47:22,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242657 2024-11-24T20:47:22,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242657 2024-11-24T20:47:22,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242657) roll requested 2024-11-24T20:47:22,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242665 2024-11-24T20:47:22,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242665 2024-11-24T20:47:22,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242665) roll requested 2024-11-24T20:47:22,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242671 2024-11-24T20:47:22,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242671 2024-11-24T20:47:22,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242671) roll requested 2024-11-24T20:47:22,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242679 2024-11-24T20:47:22,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242679 2024-11-24T20:47:22,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242679) roll requested 2024-11-24T20:47:22,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242687 2024-11-24T20:47:22,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242687 2024-11-24T20:47:22,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242687) roll requested 2024-11-24T20:47:22,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242696 2024-11-24T20:47:22,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242696 2024-11-24T20:47:22,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242696) roll requested 2024-11-24T20:47:22,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242704 2024-11-24T20:47:22,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242704 2024-11-24T20:47:22,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242704) roll requested 2024-11-24T20:47:22,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242714 2024-11-24T20:47:22,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242714 2024-11-24T20:47:22,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242714) roll requested 2024-11-24T20:47:22,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242724 2024-11-24T20:47:22,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242724 2024-11-24T20:47:22,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242724) roll requested 2024-11-24T20:47:22,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242732 2024-11-24T20:47:22,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242732 2024-11-24T20:47:22,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242732) roll requested 2024-11-24T20:47:22,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242743 2024-11-24T20:47:22,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242743 2024-11-24T20:47:22,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242743) roll requested 2024-11-24T20:47:22,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242750 2024-11-24T20:47:22,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242750 2024-11-24T20:47:22,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242750) roll requested 2024-11-24T20:47:22,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242757 2024-11-24T20:47:22,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242757 2024-11-24T20:47:22,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242757) roll requested 2024-11-24T20:47:22,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242769 2024-11-24T20:47:22,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242769 2024-11-24T20:47:22,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242769) roll requested 2024-11-24T20:47:22,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242777 2024-11-24T20:47:22,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242777 2024-11-24T20:47:22,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242777) roll requested 2024-11-24T20:47:22,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242783 2024-11-24T20:47:22,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242783 2024-11-24T20:47:22,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242783) roll requested 2024-11-24T20:47:22,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242789 2024-11-24T20:47:22,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242789 2024-11-24T20:47:22,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242789) roll requested 2024-11-24T20:47:22,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242797 2024-11-24T20:47:22,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242797 2024-11-24T20:47:22,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242797) roll requested 2024-11-24T20:47:22,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242805 2024-11-24T20:47:22,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242805 2024-11-24T20:47:22,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242805) roll requested 2024-11-24T20:47:22,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242813 2024-11-24T20:47:22,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242813 2024-11-24T20:47:22,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242813) roll requested 2024-11-24T20:47:22,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242822 2024-11-24T20:47:22,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242822 2024-11-24T20:47:22,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242822) roll requested 2024-11-24T20:47:22,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242837 2024-11-24T20:47:22,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242837 2024-11-24T20:47:22,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242837) roll requested 2024-11-24T20:47:22,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242853 2024-11-24T20:47:22,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242853 2024-11-24T20:47:22,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242853) roll requested 2024-11-24T20:47:22,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242858 2024-11-24T20:47:22,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242858 2024-11-24T20:47:22,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242858) roll requested 2024-11-24T20:47:22,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242863 2024-11-24T20:47:22,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242863 2024-11-24T20:47:22,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242863) roll requested 2024-11-24T20:47:22,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242869 2024-11-24T20:47:22,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242869 2024-11-24T20:47:22,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242869) roll requested 2024-11-24T20:47:22,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242876 2024-11-24T20:47:22,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242876 2024-11-24T20:47:22,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242876) roll requested 2024-11-24T20:47:22,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242883 2024-11-24T20:47:22,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242883 2024-11-24T20:47:22,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242883) roll requested 2024-11-24T20:47:22,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242892 2024-11-24T20:47:22,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242892 2024-11-24T20:47:22,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242892) roll requested 2024-11-24T20:47:22,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242900 2024-11-24T20:47:22,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242900 2024-11-24T20:47:22,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242900) roll requested 2024-11-24T20:47:22,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242909 2024-11-24T20:47:22,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242909 2024-11-24T20:47:22,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242909) roll requested 2024-11-24T20:47:22,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242920 2024-11-24T20:47:22,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242920 2024-11-24T20:47:22,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242920) roll requested 2024-11-24T20:47:22,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242928 2024-11-24T20:47:22,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242928 2024-11-24T20:47:22,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242928) roll requested 2024-11-24T20:47:22,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242936 2024-11-24T20:47:22,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242936 2024-11-24T20:47:22,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242936) roll requested 2024-11-24T20:47:22,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242945 2024-11-24T20:47:22,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242945 2024-11-24T20:47:22,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242945) roll requested 2024-11-24T20:47:22,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242954 2024-11-24T20:47:22,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242954 2024-11-24T20:47:22,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242954) roll requested 2024-11-24T20:47:22,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242964 2024-11-24T20:47:22,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242964 2024-11-24T20:47:22,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242964) roll requested 2024-11-24T20:47:22,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242973 2024-11-24T20:47:22,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242973 2024-11-24T20:47:22,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242973) roll requested 2024-11-24T20:47:22,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242979 2024-11-24T20:47:22,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242979 2024-11-24T20:47:22,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242979) roll requested 2024-11-24T20:47:22,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242987 2024-11-24T20:47:22,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:22,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242987 2024-11-24T20:47:22,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:22,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:22,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:22,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:22,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242987) roll requested 2024-11-24T20:47:22,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481242994 2024-11-24T20:47:23,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242994 2024-11-24T20:47:23,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481242994) roll requested 2024-11-24T20:47:23,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243002 2024-11-24T20:47:23,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481242994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243002 2024-11-24T20:47:23,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243002) roll requested 2024-11-24T20:47:23,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243011 2024-11-24T20:47:23,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243011 2024-11-24T20:47:23,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243011) roll requested 2024-11-24T20:47:23,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243021 2024-11-24T20:47:23,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243021 2024-11-24T20:47:23,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243021) roll requested 2024-11-24T20:47:23,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243031 2024-11-24T20:47:23,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243031 2024-11-24T20:47:23,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243031) roll requested 2024-11-24T20:47:23,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243039 2024-11-24T20:47:23,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243039 2024-11-24T20:47:23,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243039) roll requested 2024-11-24T20:47:23,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243047 2024-11-24T20:47:23,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243047 2024-11-24T20:47:23,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243047) roll requested 2024-11-24T20:47:23,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243056 2024-11-24T20:47:23,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243056 2024-11-24T20:47:23,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243056) roll requested 2024-11-24T20:47:23,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243064 2024-11-24T20:47:23,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243064 2024-11-24T20:47:23,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243064) roll requested 2024-11-24T20:47:23,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243070 2024-11-24T20:47:23,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243070 2024-11-24T20:47:23,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243070) roll requested 2024-11-24T20:47:23,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243076 2024-11-24T20:47:23,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243076 2024-11-24T20:47:23,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243076) roll requested 2024-11-24T20:47:23,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243084 2024-11-24T20:47:23,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243084 2024-11-24T20:47:23,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243084) roll requested 2024-11-24T20:47:23,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243092 2024-11-24T20:47:23,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243092 2024-11-24T20:47:23,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243092) roll requested 2024-11-24T20:47:23,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243100 2024-11-24T20:47:23,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243100 2024-11-24T20:47:23,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243100) roll requested 2024-11-24T20:47:23,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243108 2024-11-24T20:47:23,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243108 2024-11-24T20:47:23,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243108) roll requested 2024-11-24T20:47:23,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243116 2024-11-24T20:47:23,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243116 2024-11-24T20:47:23,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243116) roll requested 2024-11-24T20:47:23,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243124 2024-11-24T20:47:23,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243124 2024-11-24T20:47:23,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243124) roll requested 2024-11-24T20:47:23,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243131 2024-11-24T20:47:23,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243131 2024-11-24T20:47:23,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243131) roll requested 2024-11-24T20:47:23,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243139 2024-11-24T20:47:23,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243139 2024-11-24T20:47:23,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243139) roll requested 2024-11-24T20:47:23,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243146 2024-11-24T20:47:23,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243146 2024-11-24T20:47:23,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243146) roll requested 2024-11-24T20:47:23,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243153 2024-11-24T20:47:23,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243153 2024-11-24T20:47:23,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243153) roll requested 2024-11-24T20:47:23,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243161 2024-11-24T20:47:23,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243161 2024-11-24T20:47:23,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243161) roll requested 2024-11-24T20:47:23,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243170 2024-11-24T20:47:23,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243170 2024-11-24T20:47:23,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243170) roll requested 2024-11-24T20:47:23,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243185 2024-11-24T20:47:23,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243185 2024-11-24T20:47:23,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243185) roll requested 2024-11-24T20:47:23,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243193 2024-11-24T20:47:23,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243193 2024-11-24T20:47:23,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243193) roll requested 2024-11-24T20:47:23,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243198 2024-11-24T20:47:23,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243198 2024-11-24T20:47:23,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243198) roll requested 2024-11-24T20:47:23,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243204 2024-11-24T20:47:23,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243204 2024-11-24T20:47:23,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243204) roll requested 2024-11-24T20:47:23,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243212 2024-11-24T20:47:23,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243212 2024-11-24T20:47:23,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243212) roll requested 2024-11-24T20:47:23,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243225 2024-11-24T20:47:23,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243225 2024-11-24T20:47:23,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243225) roll requested 2024-11-24T20:47:23,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243233 2024-11-24T20:47:23,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243233 2024-11-24T20:47:23,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243233) roll requested 2024-11-24T20:47:23,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243241 2024-11-24T20:47:23,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243241 2024-11-24T20:47:23,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243241) roll requested 2024-11-24T20:47:23,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243247 2024-11-24T20:47:23,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243247 2024-11-24T20:47:23,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243247) roll requested 2024-11-24T20:47:23,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243253 2024-11-24T20:47:23,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243253 2024-11-24T20:47:23,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243253) roll requested 2024-11-24T20:47:23,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243260 2024-11-24T20:47:23,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243260 2024-11-24T20:47:23,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243260) roll requested 2024-11-24T20:47:23,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243266 2024-11-24T20:47:23,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243266 2024-11-24T20:47:23,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243266) roll requested 2024-11-24T20:47:23,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243273 2024-11-24T20:47:23,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243273 2024-11-24T20:47:23,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243273) roll requested 2024-11-24T20:47:23,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243279 2024-11-24T20:47:23,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243279 2024-11-24T20:47:23,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243279) roll requested 2024-11-24T20:47:23,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243285 2024-11-24T20:47:23,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243285 2024-11-24T20:47:23,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243285) roll requested 2024-11-24T20:47:23,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243293 2024-11-24T20:47:23,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243293 2024-11-24T20:47:23,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243293) roll requested 2024-11-24T20:47:23,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243299 2024-11-24T20:47:23,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243299 2024-11-24T20:47:23,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243299) roll requested 2024-11-24T20:47:23,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243305 2024-11-24T20:47:23,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243305 2024-11-24T20:47:23,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243305) roll requested 2024-11-24T20:47:23,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243312 2024-11-24T20:47:23,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243312 2024-11-24T20:47:23,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243312) roll requested 2024-11-24T20:47:23,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243319 2024-11-24T20:47:23,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243319 2024-11-24T20:47:23,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243319) roll requested 2024-11-24T20:47:23,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243325 2024-11-24T20:47:23,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243325 2024-11-24T20:47:23,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243325) roll requested 2024-11-24T20:47:23,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243333 2024-11-24T20:47:23,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243333 2024-11-24T20:47:23,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243333) roll requested 2024-11-24T20:47:23,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243339 2024-11-24T20:47:23,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243339 2024-11-24T20:47:23,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243339) roll requested 2024-11-24T20:47:23,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243349 2024-11-24T20:47:23,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243349 2024-11-24T20:47:23,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243349) roll requested 2024-11-24T20:47:23,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243356 2024-11-24T20:47:23,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243356 2024-11-24T20:47:23,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243356) roll requested 2024-11-24T20:47:23,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243362 2024-11-24T20:47:23,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243362 2024-11-24T20:47:23,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243362) roll requested 2024-11-24T20:47:23,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243368 2024-11-24T20:47:23,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243368 2024-11-24T20:47:23,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243368) roll requested 2024-11-24T20:47:23,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243377 2024-11-24T20:47:23,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243377 2024-11-24T20:47:23,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243377) roll requested 2024-11-24T20:47:23,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243383 2024-11-24T20:47:23,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243383 2024-11-24T20:47:23,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243383) roll requested 2024-11-24T20:47:23,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243390 2024-11-24T20:47:23,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243390 2024-11-24T20:47:23,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243390) roll requested 2024-11-24T20:47:23,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243396 2024-11-24T20:47:23,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243396 2024-11-24T20:47:23,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243396) roll requested 2024-11-24T20:47:23,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243402 2024-11-24T20:47:23,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243402 2024-11-24T20:47:23,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243402) roll requested 2024-11-24T20:47:23,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243408 2024-11-24T20:47:23,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243408 2024-11-24T20:47:23,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243408) roll requested 2024-11-24T20:47:23,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243413 2024-11-24T20:47:23,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243413 2024-11-24T20:47:23,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243413) roll requested 2024-11-24T20:47:23,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243420 2024-11-24T20:47:23,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243420 2024-11-24T20:47:23,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243420) roll requested 2024-11-24T20:47:23,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243425 2024-11-24T20:47:23,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243425 2024-11-24T20:47:23,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243425) roll requested 2024-11-24T20:47:23,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243433 2024-11-24T20:47:23,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243433 2024-11-24T20:47:23,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243433) roll requested 2024-11-24T20:47:23,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243438 2024-11-24T20:47:23,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243438 2024-11-24T20:47:23,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243438) roll requested 2024-11-24T20:47:23,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243445 2024-11-24T20:47:23,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243445 2024-11-24T20:47:23,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243445) roll requested 2024-11-24T20:47:23,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243451 2024-11-24T20:47:23,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243451 2024-11-24T20:47:23,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243451) roll requested 2024-11-24T20:47:23,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243457 2024-11-24T20:47:23,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243457 2024-11-24T20:47:23,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243457) roll requested 2024-11-24T20:47:23,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243469 2024-11-24T20:47:23,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243469 2024-11-24T20:47:23,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243469) roll requested 2024-11-24T20:47:23,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243476 2024-11-24T20:47:23,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243476 2024-11-24T20:47:23,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243476) roll requested 2024-11-24T20:47:23,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243485 2024-11-24T20:47:23,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243485 2024-11-24T20:47:23,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243485) roll requested 2024-11-24T20:47:23,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243491 2024-11-24T20:47:23,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243491 2024-11-24T20:47:23,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243491) roll requested 2024-11-24T20:47:23,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243498 2024-11-24T20:47:23,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243498 2024-11-24T20:47:23,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243498) roll requested 2024-11-24T20:47:23,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243504 2024-11-24T20:47:23,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243504 2024-11-24T20:47:23,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243504) roll requested 2024-11-24T20:47:23,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243510 2024-11-24T20:47:23,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243510 2024-11-24T20:47:23,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243510) roll requested 2024-11-24T20:47:23,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243516 2024-11-24T20:47:23,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243516 2024-11-24T20:47:23,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243516) roll requested 2024-11-24T20:47:23,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243522 2024-11-24T20:47:23,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243522 2024-11-24T20:47:23,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243522) roll requested 2024-11-24T20:47:23,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243529 2024-11-24T20:47:23,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243529 2024-11-24T20:47:23,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243529) roll requested 2024-11-24T20:47:23,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243535 2024-11-24T20:47:23,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243535 2024-11-24T20:47:23,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243535) roll requested 2024-11-24T20:47:23,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243543 2024-11-24T20:47:23,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243543 2024-11-24T20:47:23,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243543) roll requested 2024-11-24T20:47:23,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243548 2024-11-24T20:47:23,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243548 2024-11-24T20:47:23,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243548) roll requested 2024-11-24T20:47:23,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243553 2024-11-24T20:47:23,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243553 2024-11-24T20:47:23,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243553) roll requested 2024-11-24T20:47:23,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243559 2024-11-24T20:47:23,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243559 2024-11-24T20:47:23,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243559) roll requested 2024-11-24T20:47:23,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243566 2024-11-24T20:47:23,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243566 2024-11-24T20:47:23,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243566) roll requested 2024-11-24T20:47:23,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243574 2024-11-24T20:47:23,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243574 2024-11-24T20:47:23,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243574) roll requested 2024-11-24T20:47:23,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243583 2024-11-24T20:47:23,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243583 2024-11-24T20:47:23,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243583) roll requested 2024-11-24T20:47:23,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243592 2024-11-24T20:47:23,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243592 2024-11-24T20:47:23,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243592) roll requested 2024-11-24T20:47:23,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243600 2024-11-24T20:47:23,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243600 2024-11-24T20:47:23,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243600) roll requested 2024-11-24T20:47:23,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243607 2024-11-24T20:47:23,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243607 2024-11-24T20:47:23,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243607) roll requested 2024-11-24T20:47:23,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243614 2024-11-24T20:47:23,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243614 2024-11-24T20:47:23,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243614) roll requested 2024-11-24T20:47:23,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243623 2024-11-24T20:47:23,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243623 2024-11-24T20:47:23,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243623) roll requested 2024-11-24T20:47:23,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243633 2024-11-24T20:47:23,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243633 2024-11-24T20:47:23,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243633) roll requested 2024-11-24T20:47:23,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243642 2024-11-24T20:47:23,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243642 2024-11-24T20:47:23,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243642) roll requested 2024-11-24T20:47:23,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243648 2024-11-24T20:47:23,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243648 2024-11-24T20:47:23,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243648) roll requested 2024-11-24T20:47:23,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243656 2024-11-24T20:47:23,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243656 2024-11-24T20:47:23,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243656) roll requested 2024-11-24T20:47:23,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243665 2024-11-24T20:47:23,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243665 2024-11-24T20:47:23,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243665) roll requested 2024-11-24T20:47:23,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243673 2024-11-24T20:47:23,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243673 2024-11-24T20:47:23,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243673) roll requested 2024-11-24T20:47:23,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243678 2024-11-24T20:47:23,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243678 2024-11-24T20:47:23,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243678) roll requested 2024-11-24T20:47:23,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243684 2024-11-24T20:47:23,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243684 2024-11-24T20:47:23,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243684) roll requested 2024-11-24T20:47:23,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243690 2024-11-24T20:47:23,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243690 2024-11-24T20:47:23,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,698 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243690) roll requested 2024-11-24T20:47:23,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243699 2024-11-24T20:47:23,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243699 2024-11-24T20:47:23,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243699) roll requested 2024-11-24T20:47:23,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243708 2024-11-24T20:47:23,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243708 2024-11-24T20:47:23,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243708) roll requested 2024-11-24T20:47:23,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243713 2024-11-24T20:47:23,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243713 2024-11-24T20:47:23,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243713) roll requested 2024-11-24T20:47:23,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243720 2024-11-24T20:47:23,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243720 2024-11-24T20:47:23,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243720) roll requested 2024-11-24T20:47:23,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243730 2024-11-24T20:47:23,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243730 2024-11-24T20:47:23,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243730) roll requested 2024-11-24T20:47:23,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243737 2024-11-24T20:47:23,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243737 2024-11-24T20:47:23,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243737) roll requested 2024-11-24T20:47:23,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243742 2024-11-24T20:47:23,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243742 2024-11-24T20:47:23,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243742) roll requested 2024-11-24T20:47:23,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243748 2024-11-24T20:47:23,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243748 2024-11-24T20:47:23,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,754 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243748) roll requested 2024-11-24T20:47:23,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243754 2024-11-24T20:47:23,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243754 2024-11-24T20:47:23,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243754) roll requested 2024-11-24T20:47:23,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243760 2024-11-24T20:47:23,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243760 2024-11-24T20:47:23,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243760) roll requested 2024-11-24T20:47:23,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243769 2024-11-24T20:47:23,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243769 2024-11-24T20:47:23,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243769) roll requested 2024-11-24T20:47:23,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243775 2024-11-24T20:47:23,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243775 2024-11-24T20:47:23,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243775) roll requested 2024-11-24T20:47:23,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243780 2024-11-24T20:47:23,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243780 2024-11-24T20:47:23,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243780) roll requested 2024-11-24T20:47:23,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243788 2024-11-24T20:47:23,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243788 2024-11-24T20:47:23,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243788) roll requested 2024-11-24T20:47:23,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243797 2024-11-24T20:47:23,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243797 2024-11-24T20:47:23,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243797) roll requested 2024-11-24T20:47:23,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243803 2024-11-24T20:47:23,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243803 2024-11-24T20:47:23,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243803) roll requested 2024-11-24T20:47:23,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243809 2024-11-24T20:47:23,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243809 2024-11-24T20:47:23,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243809) roll requested 2024-11-24T20:47:23,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243814 2024-11-24T20:47:23,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243814 2024-11-24T20:47:23,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243814) roll requested 2024-11-24T20:47:23,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243819 2024-11-24T20:47:23,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243819 2024-11-24T20:47:23,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243819) roll requested 2024-11-24T20:47:23,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243826 2024-11-24T20:47:23,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243826 2024-11-24T20:47:23,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243826) roll requested 2024-11-24T20:47:23,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243833 2024-11-24T20:47:23,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243833 2024-11-24T20:47:23,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243833) roll requested 2024-11-24T20:47:23,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243842 2024-11-24T20:47:23,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243842 2024-11-24T20:47:23,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243842) roll requested 2024-11-24T20:47:23,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243851 2024-11-24T20:47:23,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243851 2024-11-24T20:47:23,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243851) roll requested 2024-11-24T20:47:23,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243860 2024-11-24T20:47:23,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243860 2024-11-24T20:47:23,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243860) roll requested 2024-11-24T20:47:23,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243870 2024-11-24T20:47:23,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243870 2024-11-24T20:47:23,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243870) roll requested 2024-11-24T20:47:23,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243879 2024-11-24T20:47:23,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243879 2024-11-24T20:47:23,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243879) roll requested 2024-11-24T20:47:23,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243888 2024-11-24T20:47:23,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243888 2024-11-24T20:47:23,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243888) roll requested 2024-11-24T20:47:23,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243897 2024-11-24T20:47:23,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243897 2024-11-24T20:47:23,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243897) roll requested 2024-11-24T20:47:23,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243908 2024-11-24T20:47:23,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243908 2024-11-24T20:47:23,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243908) roll requested 2024-11-24T20:47:23,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243917 2024-11-24T20:47:23,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243917 2024-11-24T20:47:23,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243917) roll requested 2024-11-24T20:47:23,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243924 2024-11-24T20:47:23,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243924 2024-11-24T20:47:23,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243924) roll requested 2024-11-24T20:47:23,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243930 2024-11-24T20:47:23,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243930 2024-11-24T20:47:23,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243930) roll requested 2024-11-24T20:47:23,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243938 2024-11-24T20:47:23,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243938 2024-11-24T20:47:23,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243938) roll requested 2024-11-24T20:47:23,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243946 2024-11-24T20:47:23,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243946 2024-11-24T20:47:23,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243946) roll requested 2024-11-24T20:47:23,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243957 2024-11-24T20:47:23,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243957 2024-11-24T20:47:23,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243957) roll requested 2024-11-24T20:47:23,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243963 2024-11-24T20:47:23,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243963 2024-11-24T20:47:23,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243963) roll requested 2024-11-24T20:47:23,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243969 2024-11-24T20:47:23,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243963 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243969 2024-11-24T20:47:23,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243969) roll requested 2024-11-24T20:47:23,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243977 2024-11-24T20:47:23,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243977 2024-11-24T20:47:23,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243977) roll requested 2024-11-24T20:47:23,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243986 2024-11-24T20:47:23,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:23,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243986 2024-11-24T20:47:23,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:23,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:23,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:23,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:23,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243986) roll requested 2024-11-24T20:47:23,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481243995 2024-11-24T20:47:24,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243995 2024-11-24T20:47:24,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:24,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:24,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:24,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:24,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:24,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481243995) roll requested 2024-11-24T20:47:24,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481244005 2024-11-24T20:47:24,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481243995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481244005 2024-11-24T20:47:24,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:24,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:24,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:24,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:24,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:24,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481244005) roll requested 2024-11-24T20:47:24,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481244016 2024-11-24T20:47:24,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481244005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481244016 2024-11-24T20:47:24,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:24,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:24,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T20:47:24,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:24,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:24,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732481244016) roll requested 2024-11-24T20:47:24,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732481244025 2024-11-24T20:47:24,031 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:24,033 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8fabde2134d9c90b17ff9d84ca456f90, disabling compactions & flushes 2024-11-24T20:47:24,033 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1732481213690.8fabde2134d9c90b17ff9d84ca456f90. 2024-11-24T20:47:24,033 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1732481213690.8fabde2134d9c90b17ff9d84ca456f90. 2024-11-24T20:47:24,034 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1732481213690.8fabde2134d9c90b17ff9d84ca456f90. after waiting 0 ms 2024-11-24T20:47:24,034 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1732481213690.8fabde2134d9c90b17ff9d84ca456f90. 2024-11-24T20:47:24,035 WARN [LogRoller {}] util.Shell(1106): Interrupted while joining on: Thread[Thread-5715,5,] java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Thread.join(Thread.java:1313) ~[?:?] at java.lang.Thread.join(Thread.java:1381) ~[?:?] at org.apache.hadoop.util.Shell.joinThread(Shell.java:1103) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1063) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:637) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] 2024-11-24T20:47:24,035 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1732481213690.8fabde2134d9c90b17ff9d84ca456f90. 2024-11-24T20:47:24,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,035 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1732481213690.8fabde2134d9c90b17ff9d84ca456f90. 2024-11-24T20:47:24,035 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8fabde2134d9c90b17ff9d84ca456f90: Waiting for close lock at 1732481244033Disabling compacts and flushes for region at 1732481244033Disabling writes for close at 1732481244034 (+1 ms)Writing region close event to WAL at 1732481244035 (+1 ms)Closed at 1732481244035 2024-11-24T20:47:24,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481244016 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732481244025 2024-11-24T20:47:24,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T20:47:24,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): 8fabde2134d9c90b17ff9d84ca456f90[MyCF] 2024-11-24T20:47:24,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 8fabde2134d9c90b17ff9d84ca456f90, because it is not online on us 2024-11-24T20:47:24,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,040 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:47:24,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T20:47:24,279 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1732481244025) 2024-11-24T20:47:24,280 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ac8f77c0-c42d-2735-ed47-0ae4a31466ae 2024-11-24T20:47:25,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=590 (was 545) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9834 (was 10504)